Added english words lookup

app.py

@@ -7,7 +7,9 @@ from thefuzz import process, fuzz
 import numpy as np
 import re
 from string import punctuation
-import time
+import nltk
+nltk.download('words')
+from nltk.corpus import words
 
 
 API_URL = "https://api-inference.huggingface.co/models/Dabid/abusive-tagalog-profanity-detection"
@@ -36,8 +38,8 @@ contractions = read_text('contractions', 'json')
 lookup_words = read_text('lookup_words')
 obj_pronouns = read_text('obj_pronouns')
 profanities = read_text('profanities', 'json')
+eng_words = set(words.words())
 
-loading_countdown = 0
 
 def fuzzy_lookup(tweet):
 
@@ -45,6 +47,8 @@ def fuzzy_lookup(tweet):
     lookup_profanity = np.concatenate([np.hstack(list(profanities.values())), list(profanities.keys())])
 
     for word in tweet.split():
+        if word in eng_words:
+            break
         scores = []
         matched_words = []
         word = word.strip(punctuation)
@@ -121,7 +125,6 @@ def preprocess(tweet):
 
 
 def predict(tweet):
-    global loading_countdown 
     
     preprocessed_tweet, matched_profanity = preprocess(tweet)
 

requirements.txt

@@ -1,3 +1,4 @@
 emoji
 thefuzz[speedup]
 numpy
+nltk