Update app.py

app.py

@@ -3,12 +3,16 @@ import torch
 from torch.nn.functional import softmax
 import shap
 import requests
+from bs4 import BeautifulSoup
+from sklearn.metrics.pairwise import cosine_similarity
 from transformers import RobertaTokenizer,RobertaForSequenceClassification, pipeline
 from IPython.core.display import HTML
 model_dir = 'temp'
 tokenizer = RobertaTokenizer.from_pretrained(model_dir)
 model = RobertaForSequenceClassification.from_pretrained(model_dir)
 #pipe = pipeline("text-classification", model="thugCodeNinja/robertatemp")
+tokenizer1 = RobertaTokenizer.from_pretrained('roberta-base')
+model1 = RobertaModel.from_pretrained('roberta-base')
 pipe = pipeline("text-classification",model=model,tokenizer=tokenizer)
 def process_text(input_text):
     if input_text:
@@ -34,24 +38,47 @@ def process_text(input_text):
             return data
         except Exception as e:
             return {'error': str(e)}
+    def get_article_text(url):
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.content, 'html.parser')
+            # Extract text from the article content (you may need to adjust this based on the website's structure)
+            article_text = ' '.join([p.get_text() for p in soup.find_all('p')])
+            return article_text
+    except Exception as e:
+        print(f"An error occurred: {e}")
+    return ''
     def find_plagiarism(text):
         search_results = search(text)
         if 'items' not in search_results:
             return []
         similar_articles = []
         for item in search_results['items']:
-            title = item.get('title', '')
             link = item.get('link', '')
-            similar_articles.append([ title,link])
-        return similar_articles[:5]
+            article_text = get_article_text(link)
+            if article_text:
+            # Tokenize and encode the input text and the article text
+                encoding1 = tokenizer(text, max_length=512, truncation=True, padding=True, return_tensors="pt")
+                encoding2 = tokenizer(article_text, max_length=512, truncation=True, padding=True, return_tensors="pt")
+            
+            # Calculate embeddings using the model
+                with torch.no_grad():
+                    embedding1 = model(**encoding1).last_hidden_state.mean(dim=1)
+                    embedding2 = model(**encoding2).last_hidden_state.mean(dim=1)
+            
+            # Calculate cosine similarity between the input text and the article text embeddings
+            similarity = cosine_similarity(embedding1, embedding2)[0][0]
+            similar_articles.append({'Link': link, 'Similarity': similarity})
+            similar_articles = sorted(similar_articles, key=lambda x: x['Similarity'], reverse=True)
+            threshold = 0.5  # Adjust the threshold as needed
+            similar_articles = [article for article in similar_articles if article['Similarity'] > threshold]
+            return similar_articles[:5]
 
     prediction = pipe([text])
     explainer = shap.Explainer(pipe)
     shap_values = explainer([text])
     shap_plot_html = HTML(shap.plots.text(shap_values, display=False)).data
-    # HTML(shap.plots.text(shap_values, display=False))
-    # with open('rendered.html', 'w') as file:
-    #     file.write(shap.plots.text(shap_values, display=False))
     similar_articles = find_plagiarism(text)
 
     return processed_result, prob, final_label, shap_plot_html,similar_articles