Update functions.py

functions.py

@@ -1,3 +1,17 @@
+import pandas as pd
+import numpy as np
+# preprocess
+import string
+import nltk
+from nltk.tokenize import word_tokenize
+from nltk.stem import WordNetLemmatizer
+from nltk.corpus import wordnet
+import joblib
+
+nltk.download('punkt')
+nltk.download('wordnet')
+nltk.download('averaged_perceptron_tagger')
+
 def get_wordnet_pos(treebank_tag):
     if treebank_tag.startswith('J'):
         return wordnet.ADJ
@@ -13,9 +27,10 @@ def get_wordnet_pos(treebank_tag):
 lemmatizer = WordNetLemmatizer()
 
 def preprocess_text(text):
+    stopword_list= joblib.load('stopword_list.joblib')
     text = text.lower() # lowercase text
     tokens = word_tokenize(text) # tokenize
-    filtered_words = [word for word in tokens if word.lower() not in stopword_list]
+    filtered_words = [word for word in tokens if word.lower() not in stopword_list= load.]
     lemmatized_words = [lemmatizer.lemmatize(w, get_wordnet_pos(w)) for w in filtered_words]
     lemmatized_clean = [word.translate(str.maketrans('', '', string.punctuation)) for word in lemmatized_words]
     return ' '.join(lemmatized_clean)