Update app.py

adding feature of per Word classification

app.py

@@ -13,9 +13,7 @@ import nltk
 
 def check_by_url(txt_url):
     parsed_url = urlparse(txt_url)
-    url = (
-        f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rsplit('/', 1)[0]}/"
-    )
+    url = (f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rsplit('/', 1)[0]}/")
     print(url)
 
     new_data = []
@@ -28,9 +26,7 @@ def check_by_url(txt_url):
         punctuationfree = "".join([i for i in title if i not in string.punctuation])
         return punctuationfree
 
-    css_class_to_remove = (
-        "dp-highlighter"  # Replace with the CSS class you want to remove
-    )
+    css_class_to_remove = ("dp-highlighter")  # Replace with the CSS class you want to remove
     # Find <div> tags with the specified CSS class and remove their content
     div_tags = soup.find_all(["code", "pre"])
     for div_tag in div_tags:
@@ -55,38 +51,30 @@ def check_by_url(txt_url):
         content_without_style += p_content
 
     # Replace Unicode characters in the content and remove duplicates
-    normalized_content_with_style = re.sub(
-        r"\s+", " ", content_with_style
-    )  # Remove extra spaces
-    normalized_content_with_style = normalized_content_with_style.replace(
-        "\r", ""
-    )  # Replace '\r' characters
-    normalized_content_with_style = unicodedata.normalize(
-        "NFKD", normalized_content_with_style
-    )
+    normalized_content_with_style = re.sub(r"\s+", " ", content_with_style)  # Remove extra spaces
+    normalized_content_with_style = normalized_content_with_style.replace("\r", "")  # Replace '\r' characters
+    normalized_content_with_style = unicodedata.normalize("NFKD", normalized_content_with_style)
     normalized_content_with_style = unidecode.unidecode(normalized_content_with_style)
 
-    normalized_content_without_style = re.sub(
-        r"\s+", " ", content_without_style
-    )  # Remove extra spaces
-    normalized_content_without_style = normalized_content_without_style.replace(
-        "\r", ""
-    )  # Replace '\r' characters
-    normalized_content_without_style = unicodedata.normalize(
-        "NFKD", normalized_content_without_style
-    )
-    normalized_content_without_style = unidecode.unidecode(
-        normalized_content_without_style
-    )
+    normalized_content_without_style = re.sub(r"\s+", " ", content_without_style)  # Remove extra spaces
+    normalized_content_without_style = normalized_content_without_style.replace("\r", "")  # Replace '\r' characters
+    normalized_content_without_style = unicodedata.normalize("NFKD", normalized_content_without_style)
+    normalized_content_without_style = unidecode.unidecode(normalized_content_without_style)
 
     normalized_content_with_style += normalized_content_without_style
     new_data = {"title": title, "content": normalized_content_with_style}
+#     return new_data
 
-    model = DistilBertForSequenceClassification.from_pretrained(".")
-    tokenizer = DistilBertTokenizer.from_pretrained(".")
+    model = DistilBertForSequenceClassification.from_pretrained("/content/LoadModel")
+    tokenizer = DistilBertTokenizer.from_pretrained("/content/LoadModel")
 
+    label_mapping = {1: "SFW", 0: "NSFW"} 
     test_encodings = tokenizer.encode_plus(
-        title, truncation=True, padding=True, max_length=512, return_tensors="pt"
+        title, 
+        truncation=True, 
+        padding=True, 
+        max_length=512, 
+        return_tensors="pt"
     )
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     test_input_ids = test_encodings["input_ids"].to(device)
@@ -98,9 +86,9 @@ def check_by_url(txt_url):
         logits = outputs.logits
         predicted_labels = torch.argmax(logits, dim=1)
         probabilities = F.softmax(logits, dim=1)
-        confidence_score_title = torch.max(probabilities, dim=1).values.tolist()
-        predicted_label_title = predicted_labels.item()
-
+        confidence_score_title = torch.max(probabilities, dim=1).values.tolist()        
+        predicted_label_title = label_mapping[predicted_labels.item()]
+       
     test_encodings = tokenizer.encode_plus(
         normalized_content_with_style,
         truncation=True,
@@ -116,11 +104,7 @@ def check_by_url(txt_url):
         predicted_labels = torch.argmax(logits, dim=1)
         probabilities = F.softmax(logits, dim=1)
         confidence_scores_content = torch.max(probabilities, dim=1).values.tolist()
-        predicted_label_content = predicted_labels.item()
-
-    label_mapping = {1: "SFW", 0: "NSFW"}  # 1:True 0:false
-    predicted_label_title = label_mapping[predicted_label_title]
-    predicted_label_content = label_mapping[predicted_label_content]
+        predicted_label_content = label_mapping[predicted_labels.item()]
 
     return (
         predicted_label_title,
@@ -128,35 +112,35 @@ def check_by_url(txt_url):
         predicted_label_content,
         confidence_scores_content,
         new_data,
+        #new1,
     )
 
-
 label_mapping = {1: "SFW", 0: "NSFW"}  # 1:True 0:false
-
-
+  
 def predict_2(txt_url, normalized_content_with_style):
-    (
+  (
         predicted_label_title,
         confidence_score_title,
         predicted_label_content,
         confidence_scores_content,
-        new_data,
+        new_data, 
     ) = (None, None, None, None, None)
-    predicted_label_text, confidence_score_text = None, None
 
-    if txt_url.startswith("http://") or txt_url.startswith("https://"):
-        (
+  predicted_label_text, confidence_score_text = None, None
+
+  if txt_url.startswith("http://") or txt_url.startswith("https://"):
+    (
             predicted_label_title,
             confidence_score_title,
             predicted_label_content,
             confidence_scores_content,
             new_data,
-        ) = check_by_url(txt_url)
-    elif txt_url.startswith(""):
-        model = DistilBertForSequenceClassification.from_pretrained(".")
-        tokenizer = DistilBertTokenizer.from_pretrained(".")
+      ) = check_by_url(txt_url)
+  elif txt_url.startswith(""):
+    model = DistilBertForSequenceClassification.from_pretrained("/content/LoadModel")
+    tokenizer = DistilBertTokenizer.from_pretrained("/content/LoadModel")
 
-        test_encodings = tokenizer.encode_plus(
+    test_encodings = tokenizer.encode_plus(
             normalized_content_with_style,
             truncation=True,
             padding=True,
@@ -164,11 +148,11 @@ def predict_2(txt_url, normalized_content_with_style):
             return_tensors="pt",
         )
 
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        test_input_ids = test_encodings["input_ids"].to(device)
-        test_attention_mask = test_encodings["attention_mask"].to(device)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    test_input_ids = test_encodings["input_ids"].to(device)
+    test_attention_mask = test_encodings["attention_mask"].to(device)
 
-        with torch.no_grad():
+    with torch.no_grad():
             model = model.to(device)
             model.eval()
             outputs = model(test_input_ids, attention_mask=test_attention_mask)
@@ -178,22 +162,83 @@ def predict_2(txt_url, normalized_content_with_style):
             confidence_score_text = torch.max(probabilities, dim=1).values.tolist()
             predicted_label_text = label_mapping[predicted_labels.item()]
 
-    else:
-        print("Done")
+  return (
+            predicted_label_title,
+            confidence_score_title,
+            predicted_label_content,
+            confidence_scores_content,
+            new_data,
+            predicted_label_text,
+            confidence_score_text,
+            #new,
+            )
 
-    return (
-        predicted_label_title,
-        confidence_score_title,
-        predicted_label_content,
-        confidence_scores_content,
-        new_data,
-        predicted_label_text,
-        confidence_score_text,
-    )
+def word_by_word(txt_url, normalized_content_with_style):
+  if txt_url.startswith("http://") or txt_url.startswith("https://") or txt_url.startswith(""):
+    (
+            predicted_label_title,
+            confidence_score_title,
+            predicted_label_content,
+            confidence_scores_content,
+            new_data,
+            predicted_label_text,
+            confidence_score_text,
+      ) = predict_2(txt_url, normalized_content_with_style)
+
+  model = DistilBertForSequenceClassification.from_pretrained("/content/LoadModel")
+  tokenizer = DistilBertTokenizer.from_pretrained("/content/LoadModel")      
+  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+  model = model.to(device)
+  model.eval()
+
+  new_word={}
+  content_words =[]
+  words_2 =[] 
+  if predicted_label_content=="NSFW" or predicted_label_text=="NSFW":
+        if txt_url.startswith("http://") or txt_url.startswith("https://"):
+          content_words = new_data['content'].split()
+        else:
+          words_2 = normalized_content_with_style.split()
+
+        results = []
+        for word in content_words or words_2 :          
+          encoding = tokenizer.encode_plus(
+          word,
+          truncation=True,
+          padding=True,
+          max_length=512,
+          return_tensors="pt"
+                    )
+          input_ids = encoding["input_ids"].to(device)
+          attention_mask = encoding["attention_mask"].to(device)
+          with torch.no_grad():
+            outputs = model(input_ids, attention_mask=attention_mask)
+            logits = outputs.logits
+            probabilities = F.softmax(logits, dim=1)
+            predicted_label = torch.argmax(logits, dim=1).item()
+                      #label_mapping = {1: "SFW", 0: "NSFW"}  # 1:True 0:False
+            predicted_label_word = label_mapping[predicted_label]
+            confidence_score_word = torch.max(probabilities, dim=1).values.item()
+
+            #new_word={}        
+            if predicted_label_word=="NSFW":
+              result = {"Word": word, "Label": predicted_label_word, "Confidence": confidence_score_word}
+              results.append(result)
+              new_word = json.dumps(results) 
+  return(
+      predicted_label_title,
+      confidence_score_title,
+      predicted_label_content,
+      confidence_scores_content,
+      new_data,
+      predicted_label_text,
+      confidence_score_text,
+      new_word,
+  )
 
 
 demo = gr.Interface(
-    fn=predict_2,
+    fn=word_by_word,
     inputs=[
         gr.inputs.Textbox(label="URL", placeholder="Enter URL"),
         gr.inputs.Textbox(label="Text", placeholder="Enter Text"),
@@ -204,9 +249,10 @@ demo = gr.Interface(
         gr.outputs.Textbox(label="Content_prediction"),
         gr.outputs.Textbox(label="Content_confidence_score"),
         gr.outputs.Textbox(label="Description").style(show_copy_button=True),
-        gr.outputs.Textbox(label="Text_prediction"),
+        gr.outputs.Textbox(label="Text_prediction_score"),
         gr.outputs.Textbox(label="Text_confidence_score"),
+        gr.outputs.Textbox(label="word-by-word").style(show_copy_button=True),
     ],
-)
+) 
 
-demo.launch()
+demo.launch(debug=True, share= True)