diff options
| author | Scott Gasch <[email protected]> | 2021-09-28 23:27:51 -0700 |
|---|---|---|
| committer | Scott Gasch <[email protected]> | 2021-09-28 23:27:51 -0700 |
| commit | 986d5f7ada15e56019518db43d07b76f94468e1a (patch) | |
| tree | ac3460a13667ac3dc8aa1e84c6b4fc8cfee648dc /string_utils.py | |
| parent | 0bc6e4312cad0f997751739e750954ac39dfa6cc (diff) | |
Various changes
Diffstat (limited to 'string_utils.py')
| -rw-r--r-- | string_utils.py | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/string_utils.py b/string_utils.py index 3aaf1d7..b3019cf 100644 --- a/string_utils.py +++ b/string_utils.py @@ -14,6 +14,8 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple import unicodedata from uuid import uuid4 +import list_utils + logger = logging.getLogger(__name__) NUMBER_RE = re.compile(r"^([+\-]?)((\d+)(\.\d+)?([e|E]\d+)?|\.\d+)$") @@ -1282,12 +1284,12 @@ def ngrams(txt: str, n: int): """ words = txt.split() - return ngrams_presplit(words, n) + for ngram in ngrams_presplit(words, n): + return ' '.join(ngram) def ngrams_presplit(words: Iterable[str], n: int): - for ngram in zip(*[words[i:] for i in range(n)]): - yield(' '.join(ngram)) + return list_utils.ngrams(words, n) def bigrams(txt: str): |