Update app.py

app.py

@@ -6,9 +6,7 @@ import torch
 import torch.nn.functional as F
 import re
 import logging
-import os
 import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -16,53 +14,81 @@ logger = logging.getLogger(__name__)
 
 class FakeNewsDetector:
     def __init__(self):
-        logger.info("Loading sentence transformer model directly...")
+        logger.info("Loading sentence transformer model...")
         
         try:
-            # Load model and tokenizer directly
+            # Load model and tokenizer
             self.model_name = "sentence-transformers/all-MiniLM-L6-v2"
             self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
             self.model = AutoModel.from_pretrained(self.model_name)
+            self.model.eval()
             logger.info("Model loaded successfully!")
         except Exception as e:
             logger.error(f"Error loading model: {e}")
             raise
         
-        # Fake news patterns
+        # Fake news pattern phrases (each as separate string for embedding)
         self.fake_news_patterns = [
-            "conspiracy theory", "false claim", "misinformation", "debunked",
-            "hoax", "unverified", "clickbait", "fake news", "deep state",
-            "mainstream media lies", "cover up", "they don't want you to know",
-            "secret truth", "hidden facts", "wake up people", "government lying",
-            "media conspiracy", "false flag", "planned pandemic"
+            "this is a conspiracy theory and false information",
+            "this article contains debunked misinformation and hoaxes",
+            "unverified clickbait and fake news content",
+            "deep state conspiracy and government cover up",
+            "mainstream media lies and hidden truth",
+            "they don't want you to know this secret",
+            "wake up people this is a false flag",
+            "shocking exclusive truth they are hiding",
+            "do your own research mainstream media won't report",
+            "this will shock you unbelievable secret revealed",
+            "breaking secret news the truth exposed",
+            "government lying about planned pandemic",
+            "media conspiracy and propaganda machine",
+            "open your eyes wake up sheeple"
         ]
         
+        # Pre-compute embeddings for fake news patterns
+        logger.info("Pre-computing pattern embeddings...")
+        self.pattern_embeddings = self._compute_pattern_embeddings()
+        logger.info("Pattern embeddings ready!")
+        
         # Credible sources
         self.credible_sources = [
             'reuters.com', 'apnews.com', 'bbc.com', 'nytimes.com',
             'theguardian.com', 'washingtonpost.com', 'npr.org',
             'wsj.com', 'ft.com', 'bloomberg.com', 'abcnews.go.com',
-            'cbsnews.com', 'nbcnews.com', 'cnn.com'
+            'cbsnews.com', 'nbcnews.com', 'cnn.com', 'axios.com'
         ]
         
-        # Fake news indicators
+        # Fake news text indicators
         self.fake_indicators = [
             "exclusive reveal", "shocking truth", "they don't want you to know",
-            "mainstream media won't report this", "breaking secret news",
-            "you won't believe", "this will shock you", "do your own research",
-            "the truth they're hiding", "wake up sheeple", "open your eyes"
+            "mainstream media won't report", "breaking secret", "you won't believe",
+            "this will shock you", "do your own research", "wake up sheeple",
+            "the truth they're hiding", "open your eyes", "government doesn't want",
+            "big pharma conspiracy", "new world order", "illuminati confirmed"
         ]
         
         # Sensational words
         self.sensational_words = [
-            'shocking', 'amazing', 'unbelievable', 'incredible', 'astounding',
+            'shocking', 'unbelievable', 'incredible', 'astounding',
             'mind-blowing', 'explosive', 'bombshell', 'earth-shattering',
-            'revolutionary', 'game-changing', 'miracle'
+            'miracle', 'secret', 'hidden', 'exposed', 'revealed'
         ]
 
+    def _compute_pattern_embeddings(self):
+        """Pre-compute embeddings for all fake news patterns"""
+        embeddings = []
+        for pattern in self.fake_news_patterns:
+            embedding = self.get_sentence_embedding(pattern)
+            if embedding is not None:
+                embeddings.append(embedding)
+        
+        if embeddings:
+            return torch.cat(embeddings, dim=0)
+        return None
+
     def mean_pooling(self, model_output, attention_mask):
         """Apply mean pooling to get sentence embeddings"""
-        token_embeddings = model_output[0]  # First element contains all token embeddings
+        token_embeddings = model_output[0]
         input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
         sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
         sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
@@ -73,37 +99,37 @@ class FakeNewsDetector:
         try:
             if not text or len(text.strip()) == 0:
                 return None
-                
-            # Tokenize sentences
+            
+            # Tokenize
             encoded_input = self.tokenizer(
                 text, 
                 padding=True, 
                 truncation=True, 
-                max_length=512, 
+                max_length=384,
                 return_tensors='pt'
             )
             
-            # Compute token embeddings
+            # Compute embeddings
             with torch.no_grad():
                 model_output = self.model(**encoded_input)
             
-            # Perform pooling
+            # Mean pooling
             sentence_embeddings = self.mean_pooling(model_output, encoded_input['attention_mask'])
             
-            # Normalize embeddings
+            # Normalize
             sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
             
             return sentence_embeddings
             
         except Exception as e:
-            logger.error(f"Error getting sentence embedding: {e}")
+            logger.error(f"Error getting embedding: {e}")
             return None
 
     def extract_content(self, url: str):
         """Extract content from URL"""
         try:
             headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
             }
             
             if not url.startswith(('http://', 'https://')):
@@ -125,16 +151,9 @@ class FakeNewsDetector:
             # Try multiple content selectors
             content_text = ""
             content_selectors = [
-                'article',
-                '.article-content',
-                '.post-content',
-                '.story-content',
-                '.entry-content',
-                'main',
-                '[role="main"]',
-                '.news-content',
-                '.story-body',
-                '.content'
+                'article', '.article-content', '.post-content',
+                '.story-content', '.entry-content', 'main',
+                '[role="main"]', '.news-content', '.story-body'
             ]
             
             for selector in content_selectors:
@@ -143,7 +162,7 @@ class FakeNewsDetector:
                     content_parts = []
                     for elem in elements:
                         text = elem.get_text().strip()
-                        if len(text) > 100:  # Only substantial content
+                        if len(text) > 100:
                             content_parts.append(text)
                     if content_parts:
                         content_text = ' '.join(content_parts)
@@ -171,9 +190,7 @@ class FakeNewsDetector:
 
     def clean_text(self, text: str):
         """Clean and normalize text"""
-        # Remove extra whitespace
         text = re.sub(r'\s+', ' ', text)
-        # Remove special characters but keep basic punctuation
         text = re.sub(r'[^\w\s.,!?;:()-]', '', text)
         return text.strip()
 
@@ -181,7 +198,7 @@ class FakeNewsDetector:
         """Analyze text content for fake news indicators"""
         text_lower = text.lower()
         
-        # Count various indicators
+        # Count indicators
         sensational_score = sum(1 for word in self.sensational_words if word in text_lower)
         fake_indicator_count = sum(1 for indicator in self.fake_indicators if indicator in text_lower)
         
@@ -189,11 +206,11 @@ class FakeNewsDetector:
         exclamation_count = text.count('!')
         question_count = text.count('?')
         
-        # Check for all-caps words
+        # All-caps words
         capital_words = len(re.findall(r'\b[A-Z]{3,}\b', text))
         
-        # Check for emotional language
-        emotional_words = ['outrageous', 'disgusting', 'horrible', 'terrible', 'awful']
+        # Emotional language
+        emotional_words = ['outrageous', 'disgusting', 'horrible', 'terrible', 'corrupt', 'evil']
         emotional_count = sum(1 for word in emotional_words if word in text_lower)
         
         return {
@@ -207,42 +224,60 @@ class FakeNewsDetector:
         }
 
     def check_source_credibility(self, url: str):
-        """Check if the source is known to be credible"""
+        """Check source credibility"""
         url_lower = url.lower()
         
         # Check credible sources
         for credible_source in self.credible_sources:
             if credible_source in url_lower:
-                return 0.8  # High credibility
+                return 0.85
         
-        # Penalize known unreliable domains
-        unreliable_domains = ['.blogspot.', '.wordpress.', '.tumblr.', 'medium.com']
+        # Penalize unreliable domains
+        unreliable_domains = ['.blogspot.', '.wordpress.', '.tumblr.']
         for domain in unreliable_domains:
             if domain in url_lower:
-                return 0.2  # Low credibility
+                return 0.15
                 
-        return 0.5  # Neutral credibility
+        return 0.5
 
     def semantic_similarity_analysis(self, text: str):
-        """Analyze semantic similarity with known fake news patterns"""
+        """Analyze semantic similarity with fake news patterns - FIXED"""
         try:
             if not text or len(text) < 50:
                 return 0.0
-                
-            # Get embedding for input text
-            text_embedding = self.get_sentence_embedding(text)
-            if text_embedding is None:
-                return 0.0
             
-            # Get embeddings for fake news patterns
-            pattern_embeddings = self.get_sentence_embedding(self.fake_news_patterns)
-            if pattern_embeddings is None:
+            if self.pattern_embeddings is None:
+                logger.warning("Pattern embeddings not available")
                 return 0.0
             
-            # Calculate cosine similarity
-            similarities = F.cosine_similarity(text_embedding, pattern_embeddings)
-            max_similarity = float(torch.max(similarities).item())
+            # Split text into chunks for better analysis
+            max_chunk_length = 500
+            chunks = [text[i:i+max_chunk_length] for i in range(0, min(len(text), 2000), max_chunk_length)]
+            
+            max_similarity = 0.0
+            
+            for chunk in chunks:
+                if len(chunk.strip()) < 30:
+                    continue
+                    
+                # Get embedding for chunk
+                chunk_embedding = self.get_sentence_embedding(chunk)
+                if chunk_embedding is None:
+                    continue
+                
+                # Calculate cosine similarity with all pattern embeddings
+                # chunk_embedding: [1, 384], pattern_embeddings: [N, 384]
+                similarities = F.cosine_similarity(
+                    chunk_embedding.unsqueeze(1),  # [1, 1, 384]
+                    self.pattern_embeddings.unsqueeze(0),  # [1, N, 384]
+                    dim=2
+                ).squeeze()  # [N]
+                
+                # Get maximum similarity
+                chunk_max_sim = float(torch.max(similarities).item())
+                max_similarity = max(max_similarity, chunk_max_sim)
             
+            logger.info(f"Semantic similarity: {max_similarity:.4f}")
             return max_similarity
             
         except Exception as e:
@@ -250,7 +285,7 @@ class FakeNewsDetector:
             return 0.0
 
     def detect_fake_news(self, url: str):
-        """Main fake news detection function"""
+        """Main detection function - FIXED SCORING"""
         logger.info(f"Analyzing URL: {url}")
         
         # Extract content
@@ -272,7 +307,7 @@ class FakeNewsDetector:
             return {
                 'status': '⚠️ Insufficient Content',
                 'confidence': 0.0,
-                'message': 'Not enough text content found to analyze. The article may be behind a paywall or require JavaScript.',
+                'message': 'Not enough text content to analyze.',
                 'title': title,
                 'color': 'orange'
             }
@@ -282,62 +317,74 @@ class FakeNewsDetector:
         content_analysis = self.analyze_content(full_text)
         semantic_similarity = self.semantic_similarity_analysis(full_text)
         
-        # Calculate fake news score
+        # Calculate fake news score - FIXED WEIGHTS
         fake_score = 0.0
         
-        # Source credibility (inverse weight)
-        source_factor = (1 - source_credibility) * 0.25
+        # Source credibility (40% weight)
+        source_factor = (1 - source_credibility) * 0.4
         fake_score += source_factor
         
-        # Semantic similarity with fake patterns
-        semantic_factor = semantic_similarity * 0.35
+        # Semantic similarity (40% weight) - INCREASED IMPORTANCE
+        semantic_factor = semantic_similarity * 0.4
         fake_score += semantic_factor
         
-        # Content analysis
+        # Content indicators (20% weight)
         content_factor = (
-            content_analysis['sensational_score'] * 0.05 +
-            content_analysis['fake_indicator_count'] * 0.15 +
-            min(content_analysis['exclamation_count'] * 0.02, 0.1) +
-            min(content_analysis['capital_words'] * 0.01, 0.05) +
-            min(content_analysis['emotional_count'] * 0.03, 0.05)
-        ) * 0.4
+            min(content_analysis['sensational_score'] * 0.08, 0.4) +
+            min(content_analysis['fake_indicator_count'] * 0.2, 0.6) +
+            min(content_analysis['exclamation_count'] * 0.03, 0.15) +
+            min(content_analysis['capital_words'] * 0.02, 0.1) +
+            min(content_analysis['emotional_count'] * 0.05, 0.15)
+        )
+        # Normalize content factor to 0-1 range
+        content_factor = min(content_factor, 1.0) * 0.2
         fake_score += content_factor
         
         fake_score = min(fake_score, 1.0)
         
-        # Determine result
-        if fake_score > 0.7:
+        logger.info(f"Final fake score: {fake_score:.4f}")
+        logger.info(f"  - Source factor: {source_factor:.4f}")
+        logger.info(f"  - Semantic factor: {semantic_factor:.4f}")
+        logger.info(f"  - Content factor: {content_factor:.4f}")
+        
+        # Determine result - ADJUSTED THRESHOLDS
+        if fake_score > 0.65:
             status = "🚨 Likely Fake News"
             color = "red"
-        elif fake_score > 0.5:
+        elif fake_score > 0.45:
             status = "⚠️ Suspicious Content"
             color = "orange"
-        elif fake_score > 0.3:
+        elif fake_score > 0.30:
             status = "🤔 Potentially Misleading"
             color = "yellow"
         else:
             status = "✅ Likely Credible"
             color = "green"
         
-        # Create detailed analysis message
+        # Detailed message
         message = f"""
 **📊 Detailed Analysis Results:**
 
+**Overall Fake News Score: {fake_score:.2f}/1.0**
+
 **Source Analysis:**
-- Source Credibility Score: {source_credibility:.2f}/1.0
+- Source Credibility: {source_credibility:.2f}/1.0
+- Source Factor Impact: {source_factor:.3f}
 
-**Semantic Analysis:**
-- Similarity to Fake News Patterns: {semantic_similarity:.2f}/1.0
+**AI Semantic Analysis:**
+- Similarity to Fake News Patterns: {semantic_similarity:.3f}/1.0
+- Semantic Factor Impact: {semantic_factor:.3f}
 
 **Content Analysis:**
-- Sensational Language Score: {content_analysis['sensational_score']}
-- Fake News Indicators Found: {content_analysis['fake_indicator_count']}
+- Sensational Language: {content_analysis['sensational_score']} instances
+- Fake News Indicators: {content_analysis['fake_indicator_count']} found
 - Exclamation Marks: {content_analysis['exclamation_count']}
 - ALL-CAPS Words: {content_analysis['capital_words']}
 - Emotional Language: {content_analysis['emotional_count']}
+- Content Factor Impact: {content_factor:.3f}
 
 **Content Preview:**
-{content[:400]}...
+{content[:350]}...
         """.strip()
         
         return {
@@ -354,26 +401,24 @@ detector = FakeNewsDetector()
 def analyze_url(url):
     """Gradio interface function"""
     if not url.strip():
-        return "Please enter a URL", "0%", "No analysis performed.", "No title", "gray"
+        return "Please enter a URL", "0%", "No analysis performed.", "No title"
     
     try:
         result = detector.detect_fake_news(url)
-        
         confidence_percent = f"{result['confidence'] * 100:.1f}%"
         
         return (
             result['status'],
             confidence_percent,
             result['message'],
-            result['title'],
-            result['color']
+            result['title']
         )
         
     except Exception as e:
         logger.error(f"Analysis error: {e}")
-        return "❌ Analysis Error", "0%", f"An error occurred: {str(e)}", "Error", "red"
+        return "❌ Analysis Error", "0%", f"Error: {str(e)}", "Error"
 
-# Create Gradio interface
+# Gradio interface
 with gr.Blocks(
     theme=gr.themes.Soft(),
     title="Fake News Detector",
@@ -381,19 +426,14 @@ with gr.Blocks(
     .gradio-container {
         max-width: 900px !important;
     }
-    .result-box {
-        padding: 10px;
-        border-radius: 5px;
-        margin: 5px 0;
-    }
     """
 ) as demo:
     
     gr.Markdown("""
-    # 🕵️ Fake News Detector
-    **Analyze news articles for potential fake news using AI and Semantic Analysis**
+    # 🕵️ AI-Powered Fake News Detector
+    **Analyze news articles using Sentence Transformers and Semantic Analysis**
     
-    *This tool uses sentence transformers to analyze content patterns and detect potential misinformation*
+    *Uses advanced NLP to detect misinformation patterns*
     """)
     
     with gr.Row():
@@ -401,8 +441,7 @@ with gr.Blocks(
             url_input = gr.Textbox(
                 label="📰 Enter News Article URL",
                 placeholder="https://example.com/news-article",
-                lines=1,
-                max_lines=1
+                lines=1
             )
             analyze_btn = gr.Button("🔍 Analyze Article", variant="primary", size="lg")
         
@@ -413,7 +452,7 @@ with gr.Blocks(
                     interactive=False
                 )
                 confidence_score = gr.Textbox(
-                    label="📈 Confidence Score", 
+                    label="📈 Fake News Score", 
                     interactive=False
                 )
                 article_title = gr.Textbox(
@@ -421,17 +460,14 @@ with gr.Blocks(
                     interactive=False
                 )
     
-    details_output = gr.Markdown(
-        label="📊 Detailed Analysis"
-    )
+    details_output = gr.Markdown(label="📊 Detailed Analysis")
     
-    # Examples
     gr.Examples(
         label="💡 Try these examples:",
         examples=[
-            ["https://www.bbc.com/news"],
-            ["https://www.reuters.com/"],
-            ["https://apnews.com/"]
+            ["https://www.reuters.com/world/"],
+            ["https://apnews.com/"],
+            ["https://www.bbc.com/news"]
         ],
         inputs=url_input
     )
@@ -440,18 +476,15 @@ with gr.Blocks(
     ---
     
     **🔍 How it works:**
-    1. **Content Extraction**: Extracts text from the provided URL
-    2. **Semantic Analysis**: Uses sentence transformers to analyze similarity with known fake news patterns
-    3. **Source Verification**: Checks the domain against known credible sources
-    4. **Pattern Detection**: Identifies sensational language and fake news indicators
-    5. **Confidence Scoring**: Provides a comprehensive confidence score
-    
-    **⚠️ Disclaimer**: This is an AI-powered educational tool. Always verify information through multiple credible sources and fact-checking organizations.
+    1. **Content Extraction**: Scrapes article text
+    2. **Semantic Analysis**: Compares text embeddings with fake news patterns using cosine similarity
+    3. **Source Verification**: Checks domain credibility
+    4. **Pattern Detection**: Identifies misleading language indicators
+    5. **Confidence Scoring**: Weighted score from all analyses
     
-    *Built with ❤️ using Transformers from Hugging Face*
+    **⚠️ Disclaimer**: Educational tool only. Always verify through multiple credible sources.
     """)
     
-    # Set up the analysis button
     analyze_btn.click(
         fn=analyze_url,
         inputs=url_input,
@@ -459,8 +492,4 @@ with gr.Blocks(
     )
 
 if __name__ == "__main__":
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=False
-    )
+    demo.launch(server_name="0.0.0.0", server_port=7860, share=False)