Modified reverted changes

obj_pronouns.txt → addon_words.txt

@@ -7,4 +7,8 @@ ninyo
 nila
 ka
 nyo
-ng
+ng
+an
+am
+napaka
+paka

app.py

@@ -16,8 +16,8 @@ API_URL = "https://api-inference.huggingface.co/models/Dabid/abusive-tagalog-pro
 headers = {"Authorization": "Bearer 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"}
 
 def query(text):
-    text = {"inputs": text}
-    response = requests.post(API_URL, headers=headers, json=text)
+    payload = {"inputs": text}
+    response = requests.post(API_URL, headers=headers, json=payload)
     return response.json()
 
 def read_text(filename, filetype='txt'):
@@ -35,21 +35,32 @@ def read_text(filename, filetype='txt'):
 
 
 contractions = read_text('contractions', 'json')
-lookup_words = read_text('lookup_words')
-obj_pronouns = read_text('obj_pronouns')
+similar_words = read_text('similar_words')
+addon_words = read_text('addon_words')
 profanities = read_text('profanities', 'json')
 lookup_profanity = np.concatenate([np.hstack(list(profanities.values())), list(profanities.keys())])
+lookup_words = list(set(similar_words).union(set(lookup_profanity.tolist())))
 eng_words = list(set(words.words()) - set(lookup_profanity))
 
 # TODO check eng words that are tagalog profanities
 
 def fuzzy_lookup(tweet):
 
-    matched_profanity = dict()
+    matched_profanity = []
 
     for word in tweet.split():
+
+        base_word = word
+
         if word in eng_words:
             continue
+        
+        for addon in addon_words:
+            if word.startswith(addon):
+                word[len(addon):]
+            if word.endswith(addon):
+                word[:-len(addon)]
+
         scores = []
         matched_words = []
         word = word.strip(punctuation)
@@ -65,27 +76,9 @@ def fuzzy_lookup(tweet):
             if len(scores) > 0:
                 max_score_index = np.argmax(scores)
                 if matched_words[max_score_index] in lookup_profanity:
-                    matched_profanity[word] = matched_words[max_score_index]
+                    matched_profanity.append(base_word)
 
-    # Expand Pronouns in Profanities
-    for word, profanity in matched_profanity.items():
-        word_split = word.split(profanity[-2:])
-        for pronoun in obj_pronouns:
-            if len(word_split) > 1:
-                if pronoun == word_split[-1]:
-                    matched_profanity[word] = profanity + ' ' + pronoun
-                    break
-
-    # Replace each profanities by fuzzy lookup result
-    for word, profanity in matched_profanity.items():
-        tweet = tweet.replace(word, profanity)
-
-    for profanity, prof_varations in profanities.items():
-        if len(prof_varations) > 0:
-            for prof_variant in prof_varations:
-                tweet = tweet.replace(prof_variant, profanity)
-
-    return tweet, matched_profanity
+    return matched_profanity
 
 
 def preprocess(tweet):
@@ -112,44 +105,40 @@ def preprocess(tweet):
     # Combine list of words back to sentence
     preprocessed_tweet = ' '.join(filter(None, row_split))
 
-    # Fuzzy Lookup
-    preprocessed_tweet, matches = fuzzy_lookup(preprocessed_tweet)
-
     if len(preprocessed_tweet.split()) == 1:
-        return preprocessed_tweet, matches
+        return preprocessed_tweet
 
     # Expand Contractions
     for i in contractions.items():
         preprocessed_tweet = re.sub(rf"\b{i[0]}\b", i[1], preprocessed_tweet)
 
-    return preprocessed_tweet, matches 
+    return preprocessed_tweet
 
 
 
 def predict(tweet):
     
-    preprocessed_tweet, matched_profanity = preprocess(tweet)
+    preprocessed_tweet = preprocess(tweet)
+    matched_profanity = fuzzy_lookup(preprocessed_tweet)
 
-    prediction = query(preprocessed_tweet)
+    if len(matched_profanity) > 0:
 
-    if type(prediction) == dict: 
-        print(prediction)
-        error_message = prediction['error']
-        return error_message
+        prediction = query(preprocessed_tweet)
 
-    
-    if bool(matched_profanity) == False:
-        return "No Profanity"
+        if type(prediction) == dict: 
+            print(prediction)
+            error_message = prediction['error']
+            return error_message, [[]]
         
-    
-    prediction = [tuple(i.values()) for i in prediction[0]]
-    prediction = dict((x, y) for x, y in prediction)
-    
-    print("\nTWEET:", tweet)
-    print("DETECTED PROFANITY:", matched_profanity)
-    print("LABELS:", prediction, "\n")
+        prediction = prediction[0][0]["label"]
+        
+        print("\nTWEET:", tweet)
+        print("DETECTED PROFANITY:", matched_profanity)
+        print("LABEL:", prediction, "\n")
 
-    return prediction
+        return prediction, [matched_profanity]
+    
+    return "No Profanity", [[]]
 
 
 demo = gr.Interface(
@@ -157,13 +146,18 @@ demo = gr.Interface(
 
     inputs=[gr.components.Textbox(lines=5, placeholder='Enter your input here', label='INPUT')],
 
-    outputs=[gr.components.Label(num_top_classes=2, label="PREDICTION")],
+    outputs=[gr.components.Text(label="PREDICTION"), gr.List(label="PROFANITIES")],
 
     examples=['Tangina mo naman sobrang yabang mo gago!!😠😤 @davidrafael',
               'Napakainit ngayong araw pakshet namaaan!!',
               'Napakabagal naman ng wifi tangina #PLDC #HelloDITO',
               'Bobo ka ba? napakadali lang nyan eh... 🤡',
               'Uy gago laptrip yung nangyare samen kanina HAHAHA😂😂'],
+
+    allow_flagging="never",
+
+    title="Tagalog Profanity Classifier"
 )
 
-demo.launch(debug=True)
+demo.launch(debug=True)
+predict("Tangina mo naman gag0 ka ba")

profanities.json

@@ -1,16 +1,16 @@
 {
-    "bobo": ["boboka", "b0b0"],
-    "bwiset": ["buwesit", "buwiset"],
-    "gago": ["gaga", "g@g0", "ginago"],
+    "bobo": ["bobobo", "b0b0"],
+    "bwiset": ["buwesit", "buwiset", "bwisit"],
+    "gago": ["gaga", "g@g0"],
     "kupal": [],
-    "pakshet": [],
+    "pakshet": ["pakshit"],
     "pakyu": [],
     "pucha": [],
     "punyeta": [],
     "puta": ["pota"],
     "putangina": ["pukingina", "kinangina", "putang"],
     "tanga": [],
-    "tangina": ["tangna", "inamo"],
-    "tarantado": ["tinarantado", "t@r@nt@d0"],
+    "tangina": ["tangna", "inamo", "tatanga"],
+    "tarantado": ["t4r4nt4do", "t@r@nt@do"],
     "ulol": ["ul0l", "olol", "0lol"]
 }