diff options
Diffstat (limited to 'profanity_filter.py')
| -rwxr-xr-x | profanity_filter.py | 17 |
1 files changed, 11 insertions, 6 deletions
diff --git a/profanity_filter.py b/profanity_filter.py index e5c9e11..37756ba 100755 --- a/profanity_filter.py +++ b/profanity_filter.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +"""A helper to identify and optionally obscure some bad words.""" + import logging import random import re @@ -17,6 +19,8 @@ logger = logging.getLogger(__name__) @decorator_utils.singleton class ProfanityFilter(object): + """A helper to identify and optionally obscure some bad words.""" + def __init__(self): self.bad_words = set( [ @@ -499,7 +503,8 @@ class ProfanityFilter(object): chunks = [self.stemmer.stem(word) for word in nltk.word_tokenize(result)] return ' '.join(chunks) - def tokenize(self, text: str): + @staticmethod + def tokenize(text: str): for x in nltk.word_tokenize(text): for y in re.split(r'\W+', x): yield y @@ -518,24 +523,24 @@ class ProfanityFilter(object): False """ - words = [word for word in self.tokenize(text)] + words = list(self.tokenize(text)) for word in words: if self.is_bad_word(word): - logger.debug(f'"{word}" is profanity') + logger.debug('"%s" is profanity', word) return True if len(words) > 1: for bigram in string_utils.ngrams_presplit(words, 2): bigram = ' '.join(bigram) if self.is_bad_word(bigram): - logger.debug(f'"{bigram}" is profanity') + logger.debug('"%s" is profanity', bigram) return True if len(words) > 2: for trigram in string_utils.ngrams_presplit(words, 3): trigram = ' '.join(trigram) if self.is_bad_word(trigram): - logger.debug(f'"{trigram}" is profanity') + logger.debug('"%s" is profanity', trigram) return True return False @@ -563,7 +568,7 @@ class ProfanityFilter(object): break return out - words = [x for x in self.tokenize(text)] + words = list(self.tokenize(text)) words.append('') words.append('') words.append('') |