feat: Implement and fix the news web scraping application

NewsApex

@@ -0,0 +1 @@
+Subproject commit 31d4e764b5898ca393eb0a0343bc026a052db6d7

app.py

@@ -36,15 +36,69 @@ st.markdown("""
         visibility: hidden !important;
     } */
     .main {
+        background-color: #f0f2f6;
+    }
+    .stCard {
+        background-color: white;
+        padding: 1rem;
+        border-radius: 12px;
+        border: 1px solid #e0e0e0;
+        margin-bottom: 1.5rem;
+        box-shadow: 0 4px 6px rgba(0,0,0,0.05);
+        transition: transform 0.2s ease-in-out;
+        height: 480px; /* Fixed height for alignment */
+        display: flex;
+        flex-direction: column;
+        justify-content: space-between;
+    }
+    .stCard:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 8px 15px rgba(0,0,0,0.1);
+    }
+    .card-img {
+        width: 100%;
+        height: 200px; /* Fixed image height */
+        object-fit: cover;
+        border-radius: 8px;
+        margin-bottom: 0.8rem;
         background-color: #f8f9fa;
+        display: block;
+    }
+    .card-img-placeholder {
+        width: 100%;
+        height: 200px;
+        background-color: #e9ecef;
+        border-radius: 8px;
+        margin-bottom: 0.8rem;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        color: #adb5bd;
+        font-size: 0.9rem;
+    }
+    .card-title {
+        font-size: 1.1rem;
+        font-weight: 700;
+        color: #1a1a1a;
+        margin-bottom: 0.5rem;
+        line-height: 1.4;
+        height: 4.2rem; /* Fixed height for 3 lines of text */
+        display: -webkit-box;
+        -webkit-line-clamp: 3;
+        -webkit-box-orient: vertical;
+        overflow: hidden;
+    }
+    .card-meta {
+        font-size: 0.8rem;
+        color: #6c757d;
+        margin-bottom: 0.5rem;
     }
     .stArticle {
         background-color: white;
-        padding: 2rem;
-        border-radius: 10px;
+        padding: 2.5rem;
+        border-radius: 15px;
         border: 1px solid #e9ecef;
-        margin-bottom: 2rem;
-        box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+        box-shadow: 0 10px 25px rgba(0,0,0,0.05);
     }
     .full-content {
         color: #212529;
@@ -52,9 +106,9 @@ st.markdown("""
         line-height: 1.6;
         margin-top: 1rem;
         margin-bottom: 1rem;
-        padding: 1.5rem;
-        background-color: #ffffff;
-        border-left: 5px solid #007bff;
+        padding: 0;
+        background-color: transparent;
+        border-left: none;
         white-space: pre-wrap;
     }
     .summary-content {
@@ -97,164 +151,208 @@ def cached_split_into_sentences(_service, text):
 @st.cache_data(show_spinner=False)
 def cached_rate_bias(_service, text):
     return _service.rate_bias(text)
-# ------------------------
 
-def fetch_and_display_news(query, news_service, title=None, bias_mode=False):
+def display_article_detail(article, news_service):
+    """Displays the detailed view of a selected article with bias analysis and summary."""
+    if st.button("← Back to Feed"):
+        st.session_state.selected_article = None
+        st.rerun()
+
+    url = article.get('link')
+    if not url:
+        st.error("Invalid article link.")
+        return
+
+    st.markdown('<div class="stArticle">', unsafe_allow_html=True)
+    st.title(article.get('title', 'No Title'))
+    
+    # Hero image
+    img_url = article.get('image_url')
+    if img_url:
+        st.image(img_url, use_container_width=True)
+
+    st.markdown(f"""
+        <div class="meta-info">
+            <b>Source:</b> {article.get('source_id', 'Unknown')} | 
+            <b>Date:</b> {article.get('pubDate', 'Unknown')} | 
+            <a href="{url}" target="_blank">Original Link</a>
+        </div>
+    """, unsafe_allow_html=True)
+
+    with st.spinner("🧠 Analyzing article content..."):
+        full_content = cached_get_full_content(news_service, url)
+        
+        # --- ROBUST FALLBACK LOGIC ---
+        text_to_analyze = full_content
+        status_msg = None
+        
+        # 1. Try Full Content
+        if not text_to_analyze or len(text_to_analyze.strip()) < 100:
+            # 2. Try Snippet/Description
+            text_to_analyze = article.get('snippet')
+            if text_to_analyze and len(text_to_analyze.strip()) > 20:
+                status_msg = "⚠️ Full content extraction limited. Analyzing article snippet."
+            else:
+                # 3. Last Resort: Use Title
+                text_to_analyze = article.get('title', '')
+                status_msg = "⚠️ No content found. Analyzing article headline only."
+            
+        if not text_to_analyze:
+            st.error("Could not retrieve any text for this article.")
+            return
+
+        if status_msg:
+            st.warning(status_msg)
+
+        # Display the text being analyzed
+        with st.expander("📖 View Analyzed Text", expanded=True):
+            st.markdown(f'<div style="font-size: 1rem; line-height: 1.5; color: #333;">{html.escape(text_to_analyze)}</div>', unsafe_allow_html=True)
+
+        # AI Summary
+        summary = cached_summarize_content(news_service, text_to_analyze)
+        if summary:
+            st.info(f"🤖 **AI Summary:** {summary}")
+        
+        st.divider()
+
+        # Bias Analysis
+        st.subheader("⚖️ Bias Analysis")
+        overall_bias = cached_rate_bias(news_service, text_to_analyze)
+        
+        col_b1, col_b2 = st.columns([1, 3])
+        with col_b1:
+            color = "#dc3545" if overall_bias['label'] == "Biased" else "#28a745"
+            st.markdown(f"**Overall Rating:** <span style='color:{color}; font-weight:bold; font-size:1.2rem;'>{overall_bias['label']}</span>", unsafe_allow_html=True)
+        with col_b2:
+            st.progress(overall_bias['score'], text=f"Confidence: {overall_bias['score']:.1%}")
+
+        # --- Interpretation Section ---
+        if overall_bias['label'] == "Factual":
+            st.success("✅ **Interpretation: Factual Content**\nThis article primarily uses objective language, reports verifiable events, and avoids subjective modifiers or emotional framing. It aims to inform rather than influence.")
+        else:
+            st.error("⚠️ **Interpretation: Biased Content**\nThis article contains elements that suggest a non-neutral perspective. This could include the use of loaded language, emotional appeals, or selective framing designed to influence the reader's opinion.")
+
+        o_reasoning = overall_bias.get('reasoning', 'No specific reasoning provided.')
+        st.warning(f"💡 **Analysis Reasoning:** {o_reasoning}")
+
+        st.subheader("📋 Sentence-by-Sentence Breakdown")
+        sentences = cached_split_into_sentences(news_service, text_to_analyze)
+        
+        sentence_html = ""
+        for i, sentence in enumerate(sentences, 1):
+            s_bias = cached_rate_bias(news_service, sentence)
+            s_label = s_bias.get('label', 'Factual')
+            s_reasoning = s_bias.get('reasoning', '')
+            s_color = "rgba(220, 53, 69, 0.08)" if s_label == "Biased" else "transparent"
+            
+            escaped_sentence = html.escape(sentence)
+            escaped_reasoning = html.escape(s_reasoning)
+            
+            reasoning_html = f'<div style="font-size: 0.85rem; color: #721c24; margin-top: 4px; font-style: italic;">Why? {escaped_reasoning}</div>' if s_label == "Biased" else ""
+            
+            border_style = "border-left: 4px solid #dc3545;" if s_label == "Biased" else "border-left: 4px solid #e9ecef;"
+            sentence_html += f'<div style="margin-bottom: 12px; padding: 12px; background-color: {s_color}; border-radius: 6px; {border_style}"><b>{i}.</b> {escaped_sentence} <span style="font-size: 0.8rem; color: #6c757d; margin-left: 10px; font-weight: bold;">[{s_label}]</span>{reasoning_html}</div>'
+        
+        st.markdown(f'<div class="full-content" style="border:none; padding:0;">{sentence_html}</div>', unsafe_allow_html=True)
+    
+    st.markdown('</div>', unsafe_allow_html=True)
+
+def fetch_and_display_news(query, news_service, title=None):
+    """Fetches and displays news in a grid layout."""
     if title:
         st.subheader(title)
     
-    spinner_text = f"🔍 Searching for articles about '{query}'..." if query else "🔍 Fetching latest headlines..."
+    spinner_text = f"🔍 Searching for '{query}'..." if query else "🔍 Fetching latest headlines..."
     with st.spinner(spinner_text):
         try:
-            # Use cached news fetching
             articles = cached_fetch_all_news(news_service, query, language="en")
             
             if not articles:
                 st.error("No articles found for this topic. Please try another one.")
-            else:
-                # Limit to 10 articles to avoid endless fetching/loading
-                articles = articles[:10]
-                
-                if query:
-                    st.success(f"✅ Found {len(articles)} articles!")
-                
-                found_any = False
-                for idx, article in enumerate(articles):
-                    url = article.get('link')
-                    if not url:
-                        continue
+                return
+
+            # Grid layout: 3 columns
+            articles = articles[:12]
+            cols = st.columns(3)
+            for idx, article in enumerate(articles):
+                with cols[idx % 3]:
+                    st.markdown('<div class="stCard">', unsafe_allow_html=True)
+                    
+                    # Thumbnail Image handling
+                    img_url = article.get('image_url')
+                    if img_url and img_url.startswith('http'):
+                        st.markdown(f'<img src="{img_url}" class="card-img" onerror="this.style.display=\'none\'; this.nextSibling.style.display=\'flex\';">', unsafe_allow_html=True)
+                        st.markdown('<div class="card-img-placeholder" style="display:none;">🖼️ Image Unavailable</div>', unsafe_allow_html=True)
+                    else:
+                        st.markdown('<div class="card-img-placeholder">🖼️ No Image</div>', unsafe_allow_html=True)
                     
-                    # Use cached content extraction
-                    full_content = cached_get_full_content(news_service, url)
+                    # Source and Date
+                    date_str = article.get('pubDate', 'Unknown')[:10]
+                    st.markdown(f'<div class="card-meta">{article.get("source_id", "Unknown")} • {date_str}</div>', unsafe_allow_html=True)
+                    
+                    # Title
+                    st.markdown(f'<div class="card-title">{article.get("title", "No Title")}</div>', unsafe_allow_html=True)
+                    
+                    # Analyze Button
+                    if st.button("Analyze Article", key=f"btn_{idx}", use_container_width=True):
+                        st.session_state.selected_article = article
+                        st.rerun()
+                    
+                    st.markdown('</div>', unsafe_allow_html=True)
                     
-                    if full_content:
-                        found_any = True
-                        # Use cached summarization and sentence splitting
-                        summary = cached_summarize_content(news_service, full_content)
-                        
-                        with st.container():
-                            st.markdown('<div class="stArticle">', unsafe_allow_html=True)
-                            st.header(article.get('title', 'No Title'))
-                            st.markdown(f"""
-                                <div class="meta-info">
-                                    <b>Source:</b> {article.get('source_id', 'Unknown')} | 
-                                    <b>Date:</b> {article.get('pubDate', 'Unknown')} | 
-                                    <a href="{url}" target="_blank">Original Link</a>
-                                </div>
-                            """, unsafe_allow_html=True)
-                            
-                            if bias_mode:
-                                st.subheader("⚖️ Bias Analysis")
-                                overall_bias = cached_rate_bias(news_service, full_content)
-                                
-                                # Display overall rating
-                                col_b1, col_b2 = st.columns([1, 3])
-                                with col_b1:
-                                    color = "red" if overall_bias['label'] == "Biased" else "green"
-                                    st.markdown(f"**Overall Rating:** <span style='color:{color}; font-weight:bold;'>{overall_bias['label']}</span>", unsafe_allow_html=True)
-                                with col_b2:
-                                    st.progress(overall_bias['score'], text=f"Confidence: {overall_bias['score']:.1%}")
-
-                                st.subheader("📋 Full Article (Numbered Sentences for Bias Detection)")
-                                sentences = cached_split_into_sentences(news_service, full_content)
-                                # Display sentences as a numbered list
-                                sentence_html = ""
-                                for i, sentence in enumerate(sentences, 1):
-                                    # Rate individual sentence bias
-                                    s_bias = cached_rate_bias(news_service, sentence)
-                                    s_label = s_bias.get('label', 'Factual')
-                                    s_color = "rgba(255, 0, 0, 0.1)" if s_label == "Biased" else "transparent"
-                                    
-                                    # Escape each sentence
-                                    escaped_sentence = html.escape(sentence)
-                                    # Use a single line to avoid accidental markdown code block triggers
-                                    # We access the dictionary keys outside the f-string to avoid quoting issues
-                                    sentence_html += f'<div style="margin-bottom: 8px; padding: 4px; background-color: {s_color}; border-radius: 4px;"><b>{i}.</b> {escaped_sentence} <span style="font-size: 0.8rem; color: gray; margin-left: 10px;">({s_label})</span></div>'
-                                
-                                st.markdown(f'<div class="full-content">{sentence_html}</div>', unsafe_allow_html=True)
-                            else:
-                                st.subheader("Full Article")
-                                # Escape the full content to ensure it's not cut off by HTML tags
-                                escaped_content = html.escape(full_content)
-                                st.markdown(f'<div class="full-content">{escaped_content}</div>', unsafe_allow_html=True)
-                            
-                            if summary:
-                                st.subheader("Summarization")
-                                st.markdown(f'<div class="summary-content">🤖 {summary}</div>', unsafe_allow_html=True)
-                            
-                            st.markdown('</div>', unsafe_allow_html=True)
-                            st.divider()
-                
-                if not found_any:
-                    st.warning("Could not retrieve full content for any articles. The sources might be protected or paywalled.")
         except Exception as e:
-            st.error(f"An error occurred while fetching news: {str(e)}")
+            st.error(f"Error fetching news: {str(e)}")
 
 def main():
     st.title("📰 NEXTER")
-    st.markdown("Your AI-powered gateway to the latest news from around the world.")
+    st.markdown("Modern AI-Powered News Analysis")
 
-    # Initialize session state for query and search trigger
-    if 'search_query' not in st.session_state:
-        st.session_state.search_query = ""
-    if 'trigger_search' not in st.session_state:
-        st.session_state.trigger_search = False
-    if 'is_home' not in st.session_state:
-        st.session_state.is_home = True
+    # Initialize session states
+    if 'search_query' not in st.session_state: st.session_state.search_query = ""
+    if 'selected_article' not in st.session_state: st.session_state.selected_article = None
+    if 'is_home' not in st.session_state: st.session_state.is_home = True
 
-    # Initialize Service (Cached)
     with st.sidebar:
         st.header("Search Settings")
         news_service = get_news_service()
         
-        query_input = st.text_input("Enter Topic", value=st.session_state.search_query, placeholder="e.g. Artificial Intelligence, Space, Finance")
+        query_input = st.text_input("Topic Search", value=st.session_state.search_query, placeholder="e.g. Finance, AI, Sports")
         
-        # Bias Detection Toggle
-        bias_mode = st.toggle("Bias Detection Mode", help="Enable to see articles broken down into numbered sentences.")
-
-        # Update session state if input changes manually
         if query_input != st.session_state.search_query:
             st.session_state.search_query = query_input
+            st.session_state.selected_article = None
             st.session_state.is_home = False
 
-        # Manual search button
         if st.button("Fetch News", type="primary"):
-            st.session_state.trigger_search = True
+            st.session_state.selected_article = None
             st.session_state.is_home = False
-
-        # Clear Cache button
-        if st.button("🧹 Clear Cache"):
-            st.cache_data.clear()
-            st.success("Cache cleared! Reloading...")
-            time.sleep(0.5)
             st.rerun()
 
-        # Reset to Home button
-        if st.button("🏠 Home"):
+        if st.button("🏠 Home Feed"):
             st.session_state.search_query = ""
-            st.session_state.trigger_search = False
+            st.session_state.selected_article = None
             st.session_state.is_home = True
             st.rerun()
 
-        # Model Status indicator
+        st.divider()
+        if st.button("🧹 Clear Cache"):
+            st.cache_data.clear()
+            st.success("Cache cleared!")
+            time.sleep(0.5)
+            st.rerun()
+
         if news_service.bias_model:
-            st.success("✅ Local Model Loaded")
+            st.success("✅ AI Bias Model Loaded")
         else:
-            st.info("☁️ Using Cloud Fallback")
-
-    # Handle search execution
-    if st.session_state.trigger_search and st.session_state.search_query:
-        st.session_state.is_home = False # Ensure we're not in home view
-        st.write(f"DEBUG: Searching for {st.session_state.search_query}") # DEBUG
-        fetch_and_display_news(st.session_state.search_query, news_service, title="Search Results", bias_mode=bias_mode)
-        st.session_state.trigger_search = False # Reset trigger after search
-    elif st.session_state.trigger_search and not st.session_state.search_query:
-        st.sidebar.error("Please enter a topic!")
-        st.session_state.trigger_search = False
-
-    # Home View: Latest Headlines
-    if st.session_state.is_home:
-        fetch_and_display_news(None, news_service, title="Latest Headlines", bias_mode=bias_mode)
+            st.info("☁️ Cloud Analysis Active")
+
+    # Display logic
+    if st.session_state.selected_article:
+        display_article_detail(st.session_state.selected_article, news_service)
+    elif st.session_state.is_home:
+        fetch_and_display_news(None, news_service, title="Top Headlines")
+    else:
+        fetch_and_display_news(st.session_state.search_query, news_service, title=f"Results for: {st.session_state.search_query}")
 
 
 if __name__ == "__main__":

bias_module/evaluate.py

@@ -24,21 +24,36 @@ print("Using device:", device)
 
 # -----------------------------
 # Load tokenizer and model
-tokenizer = BertTokenizer.from_pretrained(config.MODEL_NAME)
-model = BertForSequenceClassification.from_pretrained(
-    config.MODEL_NAME,
-    num_labels=2
-)
+print("Loading model and tokenizer...")
+try:
+    model_cache_dir = os.path.join(PROJECT_ROOT, "bias_module", "data", "model_cache")
+    if os.path.exists(model_cache_dir):
+        print(f"Loading from local cache: {model_cache_dir}")
+        tokenizer = BertTokenizer.from_pretrained(model_cache_dir)
+        model = BertForSequenceClassification.from_pretrained(
+            model_cache_dir,
+            num_labels=2
+        )
+    else:
+        print(f"Loading from HF Hub: {config.MODEL_NAME}")
+        tokenizer = BertTokenizer.from_pretrained(config.MODEL_NAME)
+        model = BertForSequenceClassification.from_pretrained(
+            config.MODEL_NAME,
+            num_labels=2
+        )
 
-# Correct model path regardless of current working directory
-model_path = os.path.join(PROJECT_ROOT, "models", "bert_babe.pt")
-if not os.path.exists(model_path):
-    raise FileNotFoundError(f"Model file not found at {model_path}")
+    # Correct model path regardless of current working directory
+    model_path = os.path.join(PROJECT_ROOT, "models", "bert_babe.pt")
+    if not os.path.exists(model_path):
+        raise FileNotFoundError(f"Model file not found at {model_path}")
 
-model.load_state_dict(torch.load(model_path, map_location=device))
-model.to(device)
-model.eval()
-print(f"Model loaded successfully from {model_path}")
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.to(device)
+    model.eval()
+    print(f"Model loaded successfully from {model_path}")
+except Exception as e:
+    print(f"Error loading model: {e}")
+    sys.exit(1)
 
 # -----------------------------
 # Load dataset and create dataloaders
@@ -48,11 +63,14 @@ _, test_loader = create_dataloaders(dataset, batch_size=config.BATCH_SIZE)
 # -----------------------------
 # Evaluation function
 def evaluate_model(model, test_loader):
+    print("Starting evaluation...")
     true_labels = []
     predicted_labels = []
 
     with torch.no_grad():
-        for batch in test_loader:
+        for i, batch in enumerate(test_loader):
+            if i % 10 == 0:
+                print(f"Processing batch {i}...")
             input_ids, attention_mask, labels = [x.to(device) for x in batch]
             outputs = model(input_ids=input_ids, attention_mask=attention_mask)
             predictions = torch.argmax(outputs.logits, dim=1)
@@ -60,10 +78,19 @@ def evaluate_model(model, test_loader):
             true_labels.extend(labels.cpu().numpy())
             predicted_labels.extend(predictions.cpu().numpy())
 
+    print("Calculating metrics...")
     acc = accuracy_score(true_labels, predicted_labels)
+    report = classification_report(true_labels, predicted_labels, target_names=["factual", "biased"])
+    
+    # Save to file as well as print
+    with open("evaluation_results.txt", "w") as f:
+        f.write(f"Test Accuracy: {acc:.4f}\n")
+        f.write("\nClassification Report:\n")
+        f.write(report)
+        
     print(f"\nTest Accuracy: {acc:.4f}")
     print("\nClassification Report:")
-    print(classification_report(true_labels, predicted_labels, target_names=["factual", "biased"]))
+    print(report)
 
 # -----------------------------
 # Run evaluation

bias_module/load_data.py

@@ -10,14 +10,31 @@ if PROJECT_ROOT not in sys.path:
 
 # Now we can safely import config
 import config
-from datasets import load_dataset
+from datasets import load_dataset, DatasetDict, Dataset
+import pandas as pd
 
 def load_babe_dataset():
     """
-    Load the BABE dataset from Hugging Face and split into train/test.
+    Load the BABE dataset from local parquet files if available,
+    otherwise from Hugging Face.
     Returns:
-        dataset: dict with 'train' and 'test' splits
+        dataset: DatasetDict with 'train' and 'test' splits
     """
+    local_train = os.path.join(PROJECT_ROOT, "bias_module", "data", "cache", "data", "train-00000-of-00001.parquet")
+    local_test = os.path.join(PROJECT_ROOT, "bias_module", "data", "cache", "data", "test-00000-of-00001.parquet")
+
+    if os.path.exists(local_train) and os.path.exists(local_test):
+        print("Loading BABE dataset from local parquet files...")
+        train_df = pd.read_parquet(local_train)
+        test_df = pd.read_parquet(local_test)
+        
+        dataset = DatasetDict({
+            "train": Dataset.from_pandas(train_df),
+            "test": Dataset.from_pandas(test_df)
+        })
+        return dataset
+
+    print(f"Loading BABE dataset from Hugging Face ({config.DATASET_NAME})...")
     # Load the full dataset
     dataset = load_dataset(config.DATASET_NAME)
 

evaluation_results.txt

@@ -0,0 +1,11 @@
+Test Accuracy: 0.8200
+
+Classification Report:
+              precision    recall  f1-score   support
+
+     factual       0.74      0.91      0.82       441
+      biased       0.91      0.75      0.82       559
+
+    accuracy                           0.82      1000
+   macro avg       0.83      0.83      0.82      1000
+weighted avg       0.84      0.82      0.82      1000