From 986d5f7ada15e56019518db43d07b76f94468e1a Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Tue, 28 Sep 2021 23:27:51 -0700 Subject: Various changes --- string_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'string_utils.py') diff --git a/string_utils.py b/string_utils.py index 3aaf1d7..b3019cf 100644 --- a/string_utils.py +++ b/string_utils.py @@ -14,6 +14,8 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple import unicodedata from uuid import uuid4 +import list_utils + logger = logging.getLogger(__name__) NUMBER_RE = re.compile(r"^([+\-]?)((\d+)(\.\d+)?([e|E]\d+)?|\.\d+)$") @@ -1282,12 +1284,12 @@ def ngrams(txt: str, n: int): """ words = txt.split() - return ngrams_presplit(words, n) + for ngram in ngrams_presplit(words, n): + return ' '.join(ngram) def ngrams_presplit(words: Iterable[str], n: int): - for ngram in zip(*[words[i:] for i in range(n)]): - yield(' '.join(ngram)) + return list_utils.ngrams(words, n) def bigrams(txt: str): -- cgit v1.3