Update functions_preprocess.py

functions_preprocess.py

@@ -147,3 +147,10 @@ def build_vocab(data_iter):
     vocab.set_default_index(vocab["<unk>"])
     return vocab, tokenizer
 
+def clean_text(text):
+    text = text.lower()
+    text = re.sub(r'\d+', '', text)
+    text = re.sub(r'[^\w\s]', '', text)
+    text = ' '.join([lemmatizer.lemmatize(word) for word in text.split() if word not in stop_words])
+    return text
+