summaryrefslogtreecommitdiff
path: root/string_utils.py
diff options
context:
space:
mode:
authorScott Gasch <[email protected]>2021-09-29 11:21:41 -0700
committerScott Gasch <[email protected]>2021-09-29 11:21:41 -0700
commitb2eed6fefcfa14b03916c145ad3c0435b25374d0 (patch)
tree8b416bff7c0768c8b84969e5867dee398307b112 /string_utils.py
parentd82c8377ce394cad812dc0d53829f7465b3f3f4e (diff)
Add tests on bidict. Fix bug in string_utils ngrams.
Diffstat (limited to 'string_utils.py')
-rw-r--r--string_utils.py9
1 files changed, 6 insertions, 3 deletions
diff --git a/string_utils.py b/string_utils.py
index b3019cf..78e72cc 100644
--- a/string_utils.py
+++ b/string_utils.py
@@ -10,7 +10,7 @@ import numbers
import random
import re
import string
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
import unicodedata
from uuid import uuid4
@@ -1285,10 +1285,13 @@ def ngrams(txt: str, n: int):
"""
words = txt.split()
for ngram in ngrams_presplit(words, n):
- return ' '.join(ngram)
+ ret = ''
+ for word in ngram:
+ ret += f'{word} '
+ yield ret.strip()
-def ngrams_presplit(words: Iterable[str], n: int):
+def ngrams_presplit(words: Sequence[str], n: int):
return list_utils.ngrams(words, n)