fix summarization process

app/pages/05_Dashboard.py

@@ -26,6 +26,7 @@ from components.news import (
     display_economic_calendar_widget
 )
 from utils.breaking_news_scorer import get_breaking_news_scorer
+from utils.llm_summarizer import OpenAICompatSummarizer
 
 # Import news scrapers
 try:
@@ -539,6 +540,47 @@ if fetch_errors:
     for err in fetch_errors:
         logger.warning(f"Fetch error: {err}")
 
+# Batch AI summarization after all sources are collected
+ai_summary_dfs = [
+    twitter_df,
+    reddit_df,
+    rss_all_df,
+    ai_tech_df,
+    sectoral_news_df,
+    market_events_df,
+    economic_calendar_df,
+    predictions_df,
+]
+
+summarizer = OpenAICompatSummarizer()
+if summarizer.enabled:
+    all_items = []
+    for df in ai_summary_dfs:
+        if df.empty:
+            continue
+        if "summary_raw" not in df.columns:
+            df["summary_raw"] = df.get("summary", "")
+        records = df.to_dict("records")
+        for record in records:
+            if "summary_raw" not in record:
+                record["summary_raw"] = record.get("summary", "")
+        all_items.extend(records)
+
+    if all_items:
+        with st.spinner("Summarizing news with AI..."):
+            summarizer.summarize_items(all_items, source="dashboard")
+
+        ai_map = {
+            item.get("id"): item.get("summary_ai")
+            for item in all_items
+            if item.get("id") is not None
+        }
+        for df in ai_summary_dfs:
+            if df.empty or "id" not in df.columns:
+                continue
+            df["summary_ai"] = df["id"].map(ai_map)
+            df["summary"] = df["summary_ai"].fillna(df["summary"])
+
 # Clear force refresh flag after fetching is complete
 if force_refresh:
     st.session_state.force_refresh = False
@@ -839,17 +881,6 @@ if 'fetch_errors' in locals() and fetch_errors:
             st.caption(f"• {error}")
 
 # ---- AI SUMMARY METRICS ----
-ai_summary_dfs = [
-    twitter_df,
-    reddit_df,
-    rss_all_df,
-    ai_tech_df,
-    sectoral_news_df,
-    market_events_df,
-    economic_calendar_df,
-    predictions_df,
-]
-
 total_items = sum(len(df) for df in ai_summary_dfs if not df.empty)
 ai_summarized = 0
 for df in ai_summary_dfs:

app/utils/llm_summarizer.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import time
 from typing import Dict, List, Optional, Tuple
 
 import requests
@@ -27,10 +28,11 @@ class OpenAICompatSummarizer:
         self.api_base = (api_base or os.getenv("LLM_API_BASE") or "https://researchengineering-agi.hf.space").rstrip("/")
         self.api_key = api_key if api_key is not None else os.getenv("LLM_API_KEY", "")
         self.model = model or os.getenv("LLM_MODEL", "gpt-4o-mini")
-        self.timeout = timeout or int(os.getenv("LLM_TIMEOUT", "20"))
+        self.timeout = timeout or int(os.getenv("LLM_TIMEOUT", "120"))
         self.max_items_per_request = max_items_per_request or int(os.getenv("LLM_SUMMARY_BATCH", "8"))
         self.max_chars_per_item = max_chars_per_item or int(os.getenv("LLM_SUMMARY_MAX_CHARS", "1200"))
         self.enabled = os.getenv("ENABLE_AI_SUMMARIZATION", "true").lower() in {"1", "true", "yes"}
+        self.sleep_seconds = float(os.getenv("LLM_SUMMARY_SLEEP_SECONDS", "0"))
 
         self._chat_url = f"{self.api_base}/v1/chat/completions"
 
@@ -40,6 +42,8 @@ class OpenAICompatSummarizer:
 
         candidates: List[Tuple[Dict, str]] = []
         for item in items:
+            if str(item.get("summary_ai", "")).strip():
+                continue
             text = self._build_input_text(item)
             if text:
                 candidates.append((item, text))
@@ -47,7 +51,8 @@ class OpenAICompatSummarizer:
         if not candidates:
             return items
 
-        for chunk in self._chunked(candidates, self.max_items_per_request):
+        chunks = self._chunked(candidates, self.max_items_per_request)
+        for idx, chunk in enumerate(chunks, start=1):
             texts = [text for _, text in chunk]
             summaries = self._summarize_chunk(texts, source=source)
             if not summaries:
@@ -56,6 +61,8 @@ class OpenAICompatSummarizer:
                 if summary:
                     item["summary_ai"] = summary
                     item["summary"] = summary
+            if self.sleep_seconds > 0 and idx < len(chunks):
+                time.sleep(self.sleep_seconds)
 
         return items
 

app/utils/news_cache.py

@@ -11,11 +11,6 @@ from typing import List, Dict, Optional, Callable
 
 import pandas as pd
 
-try:
-    from utils.llm_summarizer import OpenAICompatSummarizer
-except Exception:  # pragma: no cover - optional dependency
-    OpenAICompatSummarizer = None
-
 logger = logging.getLogger(__name__)
 
 
@@ -48,7 +43,6 @@ class NewsCacheManager:
             'filtered_cache': {}  # Cached filtered results
         }
         logger.info(f"NewsCacheManager initialized with {default_ttl}s TTL")
-        self.summarizer = OpenAICompatSummarizer() if OpenAICompatSummarizer else None
 
     def get_news(
         self,
@@ -95,7 +89,6 @@ class NewsCacheManager:
                 return self.cache[source]['raw_news']
 
             self._prepare_summaries(new_items)
-            self._apply_ai_summaries(new_items, source=source)
 
             # Update cache
             self._update_cache(source, new_items)
@@ -244,14 +237,6 @@ class NewsCacheManager:
             if 'summary_raw' not in item:
                 item['summary_raw'] = item.get('summary', '')
 
-    def _apply_ai_summaries(self, items: List[Dict], source: Optional[str] = None):
-        if not items or not self.summarizer or not getattr(self.summarizer, 'enabled', False):
-            return
-        try:
-            self.summarizer.summarize_items(items, source=source)
-        except Exception as exc:
-            logger.warning(f"AI summarization skipped due to error: {exc}")
-
     def get_filtered_news(
         self,
         source_df: pd.DataFrame,