Update app.py

app.py

@@ -1,11 +1,14 @@
 import gradio as gr
 import requests
 from bs4 import BeautifulSoup
-from sentence_transformers import SentenceTransformer, util
+from transformers import AutoTokenizer, AutoModel
 import torch
+import torch.nn.functional as F
 import re
 import logging
-from typing import Dict, List
+import os
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -13,23 +16,33 @@ logger = logging.getLogger(__name__)
 
 class FakeNewsDetector:
     def __init__(self):
-        logger.info("Loading Sentence Transformer model...")
-        self.model = SentenceTransformer('all-MiniLM-L6-v2')
-        logger.info("Model loaded successfully!")
+        logger.info("Loading sentence transformer model directly...")
         
-        # Enhanced fake news patterns
+        try:
+            # Load model and tokenizer directly
+            self.model_name = "sentence-transformers/all-MiniLM-L6-v2"
+            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+            self.model = AutoModel.from_pretrained(self.model_name)
+            logger.info("Model loaded successfully!")
+        except Exception as e:
+            logger.error(f"Error loading model: {e}")
+            raise
+        
+        # Fake news patterns
         self.fake_news_patterns = [
             "conspiracy theory", "false claim", "misinformation", "debunked",
-            "hoax", "unverified", "clickbait", "deep state", "fake news",
+            "hoax", "unverified", "clickbait", "fake news", "deep state",
             "mainstream media lies", "cover up", "they don't want you to know",
-            "secret truth", "hidden facts", "wake up people"
+            "secret truth", "hidden facts", "wake up people", "government lying",
+            "media conspiracy", "false flag", "planned pandemic"
         ]
         
         # Credible sources
         self.credible_sources = [
             'reuters.com', 'apnews.com', 'bbc.com', 'nytimes.com',
             'theguardian.com', 'washingtonpost.com', 'npr.org',
-            'wsj.com', 'ft.com', 'bloomberg.com'
+            'wsj.com', 'ft.com', 'bloomberg.com', 'abcnews.go.com',
+            'cbsnews.com', 'nbcnews.com', 'cnn.com'
         ]
         
         # Fake news indicators
@@ -37,23 +50,62 @@ class FakeNewsDetector:
             "exclusive reveal", "shocking truth", "they don't want you to know",
             "mainstream media won't report this", "breaking secret news",
             "you won't believe", "this will shock you", "do your own research",
-            "the truth they're hiding", "wake up sheeple"
+            "the truth they're hiding", "wake up sheeple", "open your eyes"
         ]
         
         # Sensational words
         self.sensational_words = [
             'shocking', 'amazing', 'unbelievable', 'incredible', 'astounding',
-            'mind-blowing', 'explosive', 'bombshell', 'earth-shattering'
+            'mind-blowing', 'explosive', 'bombshell', 'earth-shattering',
+            'revolutionary', 'game-changing', 'miracle'
         ]
 
-    def extract_content(self, url: str) -> Dict:
-        """Extract text content from a webpage"""
+    def mean_pooling(self, model_output, attention_mask):
+        """Apply mean pooling to get sentence embeddings"""
+        token_embeddings = model_output[0]  # First element contains all token embeddings
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+        return sum_embeddings / sum_mask
+
+    def get_sentence_embedding(self, text):
+        """Get sentence embedding using the model"""
+        try:
+            if not text or len(text.strip()) == 0:
+                return None
+                
+            # Tokenize sentences
+            encoded_input = self.tokenizer(
+                text, 
+                padding=True, 
+                truncation=True, 
+                max_length=512, 
+                return_tensors='pt'
+            )
+            
+            # Compute token embeddings
+            with torch.no_grad():
+                model_output = self.model(**encoded_input)
+            
+            # Perform pooling
+            sentence_embeddings = self.mean_pooling(model_output, encoded_input['attention_mask'])
+            
+            # Normalize embeddings
+            sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+            
+            return sentence_embeddings
+            
+        except Exception as e:
+            logger.error(f"Error getting sentence embedding: {e}")
+            return None
+
+    def extract_content(self, url: str):
+        """Extract content from URL"""
         try:
             headers = {
                 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
             }
             
-            # Validate URL
             if not url.startswith(('http://', 'https://')):
                 url = 'https://' + url
             
@@ -63,14 +115,14 @@ class FakeNewsDetector:
             soup = BeautifulSoup(response.content, 'html.parser')
             
             # Remove unwanted elements
-            for element in soup(["script", "style", "nav", "footer", "header"]):
+            for element in soup(["script", "style", "nav", "footer", "header", "aside"]):
                 element.decompose()
             
             # Extract title
             title = soup.find('title')
             title_text = title.get_text().strip() if title else "No title found"
             
-            # Try to find main content
+            # Try multiple content selectors
             content_text = ""
             content_selectors = [
                 'article',
@@ -81,7 +133,8 @@ class FakeNewsDetector:
                 'main',
                 '[role="main"]',
                 '.news-content',
-                '.story-body'
+                '.story-body',
+                '.content'
             ]
             
             for selector in content_selectors:
@@ -90,13 +143,13 @@ class FakeNewsDetector:
                     content_parts = []
                     for elem in elements:
                         text = elem.get_text().strip()
-                        if len(text) > 100:  # Only take substantial content
+                        if len(text) > 100:  # Only substantial content
                             content_parts.append(text)
                     if content_parts:
                         content_text = ' '.join(content_parts)
                         break
             
-            # Fallback to body if no specific content found
+            # Fallback to body
             if not content_text or len(content_text) < 200:
                 body = soup.find('body')
                 if body:
@@ -112,12 +165,11 @@ class FakeNewsDetector:
                 'url': url
             }
             
-        except requests.exceptions.RequestException as e:
-            return {'success': False, 'error': f"Network error: {str(e)}"}
         except Exception as e:
-            return {'success': False, 'error': f"Extraction error: {str(e)}"}
+            logger.error(f"Content extraction error: {e}")
+            return {'success': False, 'error': str(e)}
 
-    def clean_text(self, text: str) -> str:
+    def clean_text(self, text: str):
         """Clean and normalize text"""
         # Remove extra whitespace
         text = re.sub(r'\s+', ' ', text)
@@ -125,55 +177,70 @@ class FakeNewsDetector:
         text = re.sub(r'[^\w\s.,!?;:()-]', '', text)
         return text.strip()
 
-    def analyze_content(self, text: str) -> Dict:
+    def analyze_content(self, text: str):
         """Analyze text content for fake news indicators"""
         text_lower = text.lower()
         
-        # Sentiment and style analysis
+        # Count various indicators
         sensational_score = sum(1 for word in self.sensational_words if word in text_lower)
         fake_indicator_count = sum(1 for indicator in self.fake_indicators if indicator in text_lower)
         
         # Punctuation analysis
         exclamation_count = text.count('!')
         question_count = text.count('?')
+        
+        # Check for all-caps words
         capital_words = len(re.findall(r'\b[A-Z]{3,}\b', text))
         
+        # Check for emotional language
+        emotional_words = ['outrageous', 'disgusting', 'horrible', 'terrible', 'awful']
+        emotional_count = sum(1 for word in emotional_words if word in text_lower)
+        
         return {
             'sensational_score': sensational_score,
             'fake_indicator_count': fake_indicator_count,
             'exclamation_count': exclamation_count,
             'question_count': question_count,
             'capital_words': capital_words,
+            'emotional_count': emotional_count,
             'text_length': len(text)
         }
 
-    def check_source_credibility(self, url: str) -> float:
+    def check_source_credibility(self, url: str):
         """Check if the source is known to be credible"""
-        domain_credibility = 0.0
+        url_lower = url.lower()
         
+        # Check credible sources
         for credible_source in self.credible_sources:
-            if credible_source in url.lower():
-                domain_credibility = 0.8
-                break
-                
+            if credible_source in url_lower:
+                return 0.8  # High credibility
+        
         # Penalize known unreliable domains
-        unreliable_domains = ['.blogspot.', '.wordpress.', 'medium.com']
+        unreliable_domains = ['.blogspot.', '.wordpress.', '.tumblr.', 'medium.com']
         for domain in unreliable_domains:
-            if domain in url.lower():
-                domain_credibility = max(0.0, domain_credibility - 0.3)
+            if domain in url_lower:
+                return 0.2  # Low credibility
                 
-        return domain_credibility
+        return 0.5  # Neutral credibility
 
-    def semantic_analysis(self, text: str) -> float:
+    def semantic_similarity_analysis(self, text: str):
         """Analyze semantic similarity with known fake news patterns"""
         try:
             if not text or len(text) < 50:
                 return 0.0
                 
-            text_embedding = self.model.encode([text], convert_to_tensor=True)
-            pattern_embeddings = self.model.encode(self.fake_news_patterns, convert_to_tensor=True)
+            # Get embedding for input text
+            text_embedding = self.get_sentence_embedding(text)
+            if text_embedding is None:
+                return 0.0
+            
+            # Get embeddings for fake news patterns
+            pattern_embeddings = self.get_sentence_embedding(self.fake_news_patterns)
+            if pattern_embeddings is None:
+                return 0.0
             
-            similarities = util.pytorch_cos_sim(text_embedding, pattern_embeddings)
+            # Calculate cosine similarity
+            similarities = F.cosine_similarity(text_embedding, pattern_embeddings)
             max_similarity = float(torch.max(similarities).item())
             
             return max_similarity
@@ -182,7 +249,7 @@ class FakeNewsDetector:
             logger.error(f"Semantic analysis error: {e}")
             return 0.0
 
-    def detect_fake_news(self, url: str) -> Dict:
+    def detect_fake_news(self, url: str):
         """Main fake news detection function"""
         logger.info(f"Analyzing URL: {url}")
         
@@ -190,12 +257,11 @@ class FakeNewsDetector:
         content_data = self.extract_content(url)
         if not content_data['success']:
             return {
-                'is_fake': 'Error',
+                'status': '❌ Extraction Failed',
                 'confidence': 0.0,
-                'error': content_data.get('error', 'Unknown error'),
-                'details': {},
+                'message': f"Could not extract content: {content_data.get('error', 'Unknown error')}",
                 'title': 'Error',
-                'content_preview': ''
+                'color': 'red'
             }
         
         title = content_data['title']
@@ -204,18 +270,17 @@ class FakeNewsDetector:
         
         if len(content.strip()) < 100:
             return {
-                'is_fake': 'Insufficient Content',
+                'status': '⚠️ Insufficient Content',
                 'confidence': 0.0,
-                'error': 'Not enough text content found to analyze',
-                'details': {},
+                'message': 'Not enough text content found to analyze. The article may be behind a paywall or require JavaScript.',
                 'title': title,
-                'content_preview': content[:200] + '...' if len(content) > 200 else content
+                'color': 'orange'
             }
         
         # Perform analyses
         source_credibility = self.check_source_credibility(url)
         content_analysis = self.analyze_content(full_text)
-        semantic_similarity = self.semantic_analysis(full_text)
+        semantic_similarity = self.semantic_similarity_analysis(full_text)
         
         # Calculate fake news score
         fake_score = 0.0
@@ -224,7 +289,7 @@ class FakeNewsDetector:
         source_factor = (1 - source_credibility) * 0.25
         fake_score += source_factor
         
-        # Semantic similarity
+        # Semantic similarity with fake patterns
         semantic_factor = semantic_similarity * 0.35
         fake_score += semantic_factor
         
@@ -233,7 +298,8 @@ class FakeNewsDetector:
             content_analysis['sensational_score'] * 0.05 +
             content_analysis['fake_indicator_count'] * 0.15 +
             min(content_analysis['exclamation_count'] * 0.02, 0.1) +
-            min(content_analysis['capital_words'] * 0.01, 0.05)
+            min(content_analysis['capital_words'] * 0.01, 0.05) +
+            min(content_analysis['emotional_count'] * 0.03, 0.05)
         ) * 0.4
         fake_score += content_factor
         
@@ -241,30 +307,45 @@ class FakeNewsDetector:
         
         # Determine result
         if fake_score > 0.7:
-            result = "🚨 Likely Fake News"
+            status = "🚨 Likely Fake News"
             color = "red"
         elif fake_score > 0.5:
-            result = "⚠️ Suspicious Content"
+            status = "⚠️ Suspicious Content"
             color = "orange"
         elif fake_score > 0.3:
-            result = "🤔 Potentially Misleading"
+            status = "🤔 Potentially Misleading"
             color = "yellow"
         else:
-            result = "✅ Likely Credible"
+            status = "✅ Likely Credible"
             color = "green"
         
+        # Create detailed analysis message
+        message = f"""
+**📊 Detailed Analysis Results:**
+
+**Source Analysis:**
+- Source Credibility Score: {source_credibility:.2f}/1.0
+
+**Semantic Analysis:**
+- Similarity to Fake News Patterns: {semantic_similarity:.2f}/1.0
+
+**Content Analysis:**
+- Sensational Language Score: {content_analysis['sensational_score']}
+- Fake News Indicators Found: {content_analysis['fake_indicator_count']}
+- Exclamation Marks: {content_analysis['exclamation_count']}
+- ALL-CAPS Words: {content_analysis['capital_words']}
+- Emotional Language: {content_analysis['emotional_count']}
+
+**Content Preview:**
+{content[:400]}...
+        """.strip()
+        
         return {
-            'is_fake': result,
+            'status': status,
             'confidence': fake_score,
-            'color': color,
-            'details': {
-                'title': title,
-                'content_preview': content[:300] + '...' if len(content) > 300 else content,
-                'source_credibility': source_credibility,
-                'semantic_similarity': semantic_similarity,
-                'content_analysis': content_analysis,
-                'url_analyzed': url
-            }
+            'message': message,
+            'title': title,
+            'color': color
         }
 
 # Initialize detector
@@ -272,95 +353,114 @@ detector = FakeNewsDetector()
 
 def analyze_url(url):
     """Gradio interface function"""
-    if not url:
-        return "Please enter a URL", "", "", "", "white"
+    if not url.strip():
+        return "Please enter a URL", "0%", "No analysis performed.", "No title", "gray"
     
     try:
         result = detector.detect_fake_news(url)
         
-        if 'error' in result and result['is_fake'] in ['Error', 'Insufficient Content']:
-            return result['is_fake'], f"Error: {result.get('error', 'Unknown error')}", "", "", "white"
-        
-        # Format output
         confidence_percent = f"{result['confidence'] * 100:.1f}%"
         
-        details_text = f"""
-**📊 Detailed Analysis Results:**
-
-**Source Analysis:**
-- Source Credibility Score: {result['details']['source_credibility']:.2f}/1.0
-
-**Content Analysis:**
-- Semantic Similarity to Fake Patterns: {result['details']['semantic_similarity']:.2f}/1.0
-- Sensational Language Score: {result['details']['content_analysis']['sensational_score']}
-- Fake News Indicators Found: {result['details']['content_analysis']['fake_indicator_count']}
-- Exclamation Marks: {result['details']['content_analysis']['exclamation_count']}
-- ALL-CAPS Words: {result['details']['content_analysis']['capital_words']}
-
-**Content Preview:**
-{result['details']['content_preview']}
-        """
-        
-        return result['is_fake'], confidence_percent, details_text, result['details']['title'], result['color']
+        return (
+            result['status'],
+            confidence_percent,
+            result['message'],
+            result['title'],
+            result['color']
+        )
         
     except Exception as e:
         logger.error(f"Analysis error: {e}")
-        return "Error", f"An error occurred during analysis: {str(e)}", "", "", "white"
+        return "❌ Analysis Error", "0%", f"An error occurred: {str(e)}", "Error", "red"
 
 # Create Gradio interface
-with gr.Blocks(theme=gr.themes.Soft(), title="Fake News Detector") as demo:
+with gr.Blocks(
+    theme=gr.themes.Soft(),
+    title="Fake News Detector",
+    css="""
+    .gradio-container {
+        max-width: 900px !important;
+    }
+    .result-box {
+        padding: 10px;
+        border-radius: 5px;
+        margin: 5px 0;
+    }
+    """
+) as demo:
+    
     gr.Markdown("""
     # 🕵️ Fake News Detector
-    **Enter a URL to analyze news articles for potential fake news indicators**
+    **Analyze news articles for potential fake news using AI and Semantic Analysis**
     
-    This tool uses AI to analyze content patterns, source credibility, and semantic similarities to detect potential fake news.
+    *This tool uses sentence transformers to analyze content patterns and detect potential misinformation*
     """)
     
     with gr.Row():
-        with gr.Column():
+        with gr.Column(scale=2):
             url_input = gr.Textbox(
-                label="Enter News Article URL",
+                label="📰 Enter News Article URL",
                 placeholder="https://example.com/news-article",
-                lines=1
+                lines=1,
+                max_lines=1
             )
-            analyze_btn = gr.Button("Analyze Article", variant="primary")
+            analyze_btn = gr.Button("🔍 Analyze Article", variant="primary", size="lg")
         
-        with gr.Column():
-            result_status = gr.Textbox(label="Analysis Result", interactive=False)
-            confidence_score = gr.Textbox(label="Confidence Score", interactive=False)
-            article_title = gr.Textbox(label="Article Title", interactive=False)
+        with gr.Column(scale=1):
+            with gr.Group():
+                result_status = gr.Textbox(
+                    label="🎯 Analysis Result",
+                    interactive=False
+                )
+                confidence_score = gr.Textbox(
+                    label="📈 Confidence Score", 
+                    interactive=False
+                )
+                article_title = gr.Textbox(
+                    label="📝 Article Title",
+                    interactive=False
+                )
     
-    with gr.Row():
-        details_output = gr.Markdown(label="Detailed Analysis")
-    
-    # Set up event handling
-    analyze_btn.click(
-        fn=analyze_url,
-        inputs=url_input,
-        outputs=[result_status, confidence_score, details_output, article_title]
+    details_output = gr.Markdown(
+        label="📊 Detailed Analysis"
     )
     
-    # Example for quick testing
+    # Examples
     gr.Examples(
+        label="💡 Try these examples:",
         examples=[
-            ["https://www.reuters.com/world/"],
-            ["https://apnews.com/"],
-            ["https://www.bbc.com/news"]
+            ["https://www.bbc.com/news"],
+            ["https://www.reuters.com/"],
+            ["https://apnews.com/"]
         ],
         inputs=url_input
     )
     
     gr.Markdown("""
     ---
-    **How it works:**
-    1. 🔍 Extracts content from the provided URL
-    2. 🤖 Analyzes text using Sentence Transformers
-    3. 📊 Checks for fake news patterns and indicators
-    4. ⚖️ Evaluates source credibility
-    5. 📈 Provides confidence score and detailed analysis
     
-    **Note:** This is an AI-powered tool and should be used as a supplementary resource for fact-checking.
+    **🔍 How it works:**
+    1. **Content Extraction**: Extracts text from the provided URL
+    2. **Semantic Analysis**: Uses sentence transformers to analyze similarity with known fake news patterns
+    3. **Source Verification**: Checks the domain against known credible sources
+    4. **Pattern Detection**: Identifies sensational language and fake news indicators
+    5. **Confidence Scoring**: Provides a comprehensive confidence score
+    
+    **⚠️ Disclaimer**: This is an AI-powered educational tool. Always verify information through multiple credible sources and fact-checking organizations.
+    
+    *Built with ❤️ using Transformers from Hugging Face*
     """)
+    
+    # Set up the analysis button
+    analyze_btn.click(
+        fn=analyze_url,
+        inputs=url_input,
+        outputs=[result_status, confidence_score, details_output, article_title]
+    )
 
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False
+    )