File size: 869 Bytes
e62ebcf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | # text_utils.py
import nltk
import string
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
ps = PorterStemmer()
def ensure_nltk_downloads():
try:
stopwords.words("english")
except LookupError:
nltk.download("stopwords")
try:
word_tokenize("Hello world")
except LookupError:
nltk.download("punkt")
ensure_nltk_downloads()
def apply_text_cleaner(x):
return x.apply(text_cleaner_func)
def text_cleaner_func(text):
text = text.lower()
words = word_tokenize(text)
words = [word for word in words if word.isalnum()]
words = [
word
for word in words
if word not in stopwords.words("english") and word not in string.punctuation
]
words = [ps.stem(word) for word in words]
return " ".join(words)
|