From b2eed6fefcfa14b03916c145ad3c0435b25374d0 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Wed, 29 Sep 2021 11:21:41 -0700 Subject: Add tests on bidict. Fix bug in string_utils ngrams. --- string_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'string_utils.py') diff --git a/string_utils.py b/string_utils.py index b3019cf..78e72cc 100644 --- a/string_utils.py +++ b/string_utils.py @@ -10,7 +10,7 @@ import numbers import random import re import string -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple import unicodedata from uuid import uuid4 @@ -1285,10 +1285,13 @@ def ngrams(txt: str, n: int): """ words = txt.split() for ngram in ngrams_presplit(words, n): - return ' '.join(ngram) + ret = '' + for word in ngram: + ret += f'{word} ' + yield ret.strip() -def ngrams_presplit(words: Iterable[str], n: int): +def ngrams_presplit(words: Sequence[str], n: int): return list_utils.ngrams(words, n) -- cgit v1.3