Update deliverable2.py

deliverable2.py

@@ -26,7 +26,8 @@ class URLValidator:
             response.raise_for_status()
             soup = BeautifulSoup(response.text, "html.parser")
             return " ".join([p.text for p in soup.find_all("p")])  # Extract paragraph text
-        except requests.RequestException:
+        except requests.RequestException as e:
+            print(f"Error fetching content from {url}: {e}")
             return ""  # Fail gracefully by returning an empty string
 
     def get_domain_trust(self, url: str, content: str) -> int:
@@ -36,7 +37,8 @@ class URLValidator:
         if content:
             try:
                 trust_scores.append(self.get_domain_trust_huggingface(content))
-            except:
+            except Exception as e:
+                print(f"Error in domain trust computation: {e}")
                 pass
 
         return int(sum(trust_scores) / len(trust_scores)) if trust_scores else 50
@@ -45,4 +47,48 @@ class URLValidator:
         """ Uses a Hugging Face fake news detection model to assess credibility. """
         if not content:
             return 50
-        result = s
+        try:
+            result = self.fake_news_classifier(content)[0]
+            if result['label'] == 'FAKE':
+                return 20  # Fake content detected
+            elif result['label'] == 'REAL':
+                return 80  # Real content detected
+            else:
+                return 50  # Neutral if unsure
+        except Exception as e:
+            print(f"Error in fake news detection: {e}")
+            return 50  # Return neutral if an error occurs
+
+    def get_content_relevance(self, query: str, content: str) -> float:
+        """ Measures content relevance to a query using Sentence Transformers. """
+        if not content:
+            return 0.0
+        query_embedding = self.similarity_model.encode(query, convert_to_tensor=True)
+        content_embedding = self.similarity_model.encode(content, convert_to_tensor=True)
+        similarity = util.pytorch_cos_sim(query_embedding, content_embedding)
+        return float(similarity)
+
+    def evaluate_url(self, url: str, query: str) -> dict:
+        """ Combines various methods to evaluate the overall credibility of a URL. """
+        content = self.fetch_page_content(url)
+        if not content:
+            return {"URL": url, "Validity": "Invalid", "Trust": 50, "Relevance": 0.0}
+
+        trust = self.get_domain_trust(url, content)
+        relevance = self.get_content_relevance(query, content)
+
+        # Decide if the URL is credible based on trust and relevance thresholds
+        validity = "Valid" if trust > 60 and relevance > 0.5 else "Invalid"
+
+        return {"URL": url, "Validity": validity, "Trust": trust, "Relevance": relevance}
+
+# Example usage
+serpapi_key = os.getenv("SERPAPI_API_KEY")  # Set your API key
+url_validator = URLValidator(serpapi_key)
+
+# Query to evaluate URL relevance
+query = "How blockchain works"
+url = "https://www.ibm.com/topics/what-is-blockchain"
+
+evaluation = url_validator.evaluate_url(url, query)
+print(evaluation)