summaryrefslogtreecommitdiff
path: root/string_utils.py
diff options
context:
space:
mode:
Diffstat (limited to 'string_utils.py')
-rw-r--r--string_utils.py20
1 files changed, 19 insertions, 1 deletions
diff --git a/string_utils.py b/string_utils.py
index 6fc257d..45cf5aa 100644
--- a/string_utils.py
+++ b/string_utils.py
@@ -9,7 +9,7 @@ import logging
import random
import re
import string
-from typing import Any, Callable, List, Optional
+from typing import Any, Callable, Iterable, List, Optional
import unicodedata
from uuid import uuid4
@@ -963,3 +963,21 @@ def thify(n: int) -> str:
return "rd"
else:
return "th"
+
+
+def ngrams(txt: str, n: int):
+ words = txt.split()
+ return ngrams_presplit(words, n)
+
+
+def ngrams_presplit(words: Iterable[str], n: int):
+ for ngram in zip(*[words[i:] for i in range(n)]):
+ yield(' '.join(ngram))
+
+
+def bigrams(txt: str):
+ return ngrams(txt, 2)
+
+
+def trigrams(txt: str):
+ return ngrams(txt, 3)