diff options
Diffstat (limited to 'string_utils.py')
| -rw-r--r-- | string_utils.py | 20 |
1 files changed, 19 insertions, 1 deletions
diff --git a/string_utils.py b/string_utils.py index 6fc257d..45cf5aa 100644 --- a/string_utils.py +++ b/string_utils.py @@ -9,7 +9,7 @@ import logging import random import re import string -from typing import Any, Callable, List, Optional +from typing import Any, Callable, Iterable, List, Optional import unicodedata from uuid import uuid4 @@ -963,3 +963,21 @@ def thify(n: int) -> str: return "rd" else: return "th" + + +def ngrams(txt: str, n: int): + words = txt.split() + return ngrams_presplit(words, n) + + +def ngrams_presplit(words: Iterable[str], n: int): + for ngram in zip(*[words[i:] for i in range(n)]): + yield(' '.join(ngram)) + + +def bigrams(txt: str): + return ngrams(txt, 2) + + +def trigrams(txt: str): + return ngrams(txt, 3) |
