add news summarization by ai

app/pages/05_Dashboard.py

@@ -838,6 +838,42 @@ if 'fetch_errors' in locals() and fetch_errors:
         for error in fetch_errors:
             st.caption(f"• {error}")
 
+# ---- AI SUMMARY METRICS ----
+ai_summary_dfs = [
+    twitter_df,
+    reddit_df,
+    rss_all_df,
+    ai_tech_df,
+    sectoral_news_df,
+    market_events_df,
+    economic_calendar_df,
+    predictions_df,
+]
+
+total_items = sum(len(df) for df in ai_summary_dfs if not df.empty)
+ai_summarized = 0
+for df in ai_summary_dfs:
+    if df.empty or "summary_ai" not in df.columns:
+        continue
+    ai_summarized += df["summary_ai"].fillna("").astype(str).str.strip().ne("").sum()
+
+ai_summary_pct = (ai_summarized / total_items * 100) if total_items else 0.0
+
+st.markdown("---")
+st.markdown("## 🤖 AI Summary")
+st.markdown(
+    f"""
+    <div style="background: linear-gradient(135deg, #1E222D 0%, #131722 100%); border: 1px solid #2A2E39; border-radius: 8px; padding: 20px; margin-bottom: 12px;">
+        <div style="color: #E0E3EB; font-size: 16px; font-weight: 600; margin-bottom: 6px;">Current AI Summarizations</div>
+        <div style="color: #D1D4DC; font-size: 14px; line-height: 1.6;">
+            {ai_summarized} / {total_items} items summarized
+            <span style="color: #787B86; font-size: 12px; margin-left: 8px;">({ai_summary_pct:.1f}% coverage)</span>
+        </div>
+    </div>
+    """,
+    unsafe_allow_html=True,
+)
+
 # Auto-refresh logic
 if auto_refresh:
     import time

app/utils/llm_summarizer.py

@@ -0,0 +1,149 @@
+"""OpenAI-compatible LLM summarizer for news items."""
+
+import json
+import logging
+import os
+from typing import Dict, List, Optional, Tuple
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAICompatSummarizer:
+    """
+    Summarize news items using an OpenAI-compatible chat completions API.
+    """
+
+    def __init__(
+        self,
+        api_base: Optional[str] = None,
+        api_key: Optional[str] = None,
+        model: Optional[str] = None,
+        timeout: Optional[int] = None,
+        max_items_per_request: Optional[int] = None,
+        max_chars_per_item: Optional[int] = None,
+    ):
+        self.api_base = (api_base or os.getenv("LLM_API_BASE") or "https://researchengineering-agi.hf.space").rstrip("/")
+        self.api_key = api_key if api_key is not None else os.getenv("LLM_API_KEY", "")
+        self.model = model or os.getenv("LLM_MODEL", "gpt-4o-mini")
+        self.timeout = timeout or int(os.getenv("LLM_TIMEOUT", "20"))
+        self.max_items_per_request = max_items_per_request or int(os.getenv("LLM_SUMMARY_BATCH", "8"))
+        self.max_chars_per_item = max_chars_per_item or int(os.getenv("LLM_SUMMARY_MAX_CHARS", "1200"))
+        self.enabled = os.getenv("ENABLE_AI_SUMMARIZATION", "true").lower() in {"1", "true", "yes"}
+
+        self._chat_url = f"{self.api_base}/v1/chat/completions"
+
+    def summarize_items(self, items: List[Dict], source: Optional[str] = None) -> List[Dict]:
+        if not self.enabled or not items:
+            return items
+
+        candidates: List[Tuple[Dict, str]] = []
+        for item in items:
+            text = self._build_input_text(item)
+            if text:
+                candidates.append((item, text))
+
+        if not candidates:
+            return items
+
+        for chunk in self._chunked(candidates, self.max_items_per_request):
+            texts = [text for _, text in chunk]
+            summaries = self._summarize_chunk(texts, source=source)
+            if not summaries:
+                continue
+            for (item, _), summary in zip(chunk, summaries):
+                if summary:
+                    item["summary_ai"] = summary
+                    item["summary"] = summary
+
+        return items
+
+    def _build_input_text(self, item: Dict) -> str:
+        title = str(item.get("title", "")).strip()
+        summary = str(item.get("summary_raw", item.get("summary", ""))).strip()
+        extra = str(item.get("content", item.get("text", item.get("description", "")))).strip()
+
+        parts = []
+        if title:
+            parts.append(f"Title: {title}")
+        if summary and summary != title:
+            parts.append(f"Summary: {summary}")
+        if extra and extra not in summary and extra not in title:
+            parts.append(f"Details: {extra}")
+
+        combined = "\n".join(parts).strip()
+        if not combined:
+            return ""
+
+        if len(combined) > self.max_chars_per_item:
+            combined = combined[: self.max_chars_per_item].rstrip()
+
+        return combined
+
+    def _summarize_chunk(self, texts: List[str], source: Optional[str] = None) -> List[str]:
+        system_prompt = (
+            "You are a financial news summarizer. "
+            "Return concise, factual summaries in 1-2 sentences, <=240 characters each. "
+            "Do not add speculation or new facts."
+        )
+        source_line = f"Source: {source}" if source else ""
+
+        items_text = []
+        for idx, text in enumerate(texts, start=1):
+            items_text.append(f"{idx}. {text}")
+
+        user_prompt = (
+            "Summarize each item below. "
+            "Return a JSON array of strings in the same order. "
+            "No extra text.\n"
+            f"{source_line}\n\n" + "\n\n".join(items_text)
+        )
+
+        payload = {
+            "model": self.model,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt},
+            ],
+            "temperature": 0.2,
+        }
+
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        try:
+            response = requests.post(self._chat_url, json=payload, headers=headers, timeout=self.timeout)
+            response.raise_for_status()
+            data = response.json()
+            content = (
+                data.get("choices", [{}])[0]
+                .get("message", {})
+                .get("content", "")
+                .strip()
+            )
+            summaries = self._parse_json_array(content)
+            if summaries and len(summaries) == len(texts):
+                return summaries
+            logger.warning("LLM summarizer returned unexpected format or length")
+            return []
+        except Exception as exc:
+            logger.warning(f"LLM summarization failed: {exc}")
+            return []
+
+    def _parse_json_array(self, content: str) -> List[str]:
+        if not content:
+            return []
+        try:
+            parsed = json.loads(content)
+            if isinstance(parsed, list):
+                return [str(x).strip() for x in parsed]
+            return []
+        except Exception:
+            return []
+
+    def _chunked(self, items: List[Tuple[Dict, str]], size: int) -> List[List[Tuple[Dict, str]]]:
+        if size <= 0:
+            return [items]
+        return [items[i : i + size] for i in range(0, len(items), size)]

app/utils/news_cache.py

@@ -6,10 +6,16 @@ Centralized cache manager for Twitter, Reddit, RSS, and AI/Tech news feeds
 import hashlib
 import logging
 import re
-import pandas as pd
 from datetime import datetime, timedelta
 from typing import List, Dict, Optional, Callable
 
+import pandas as pd
+
+try:
+    from utils.llm_summarizer import OpenAICompatSummarizer
+except Exception:  # pragma: no cover - optional dependency
+    OpenAICompatSummarizer = None
+
 logger = logging.getLogger(__name__)
 
 
@@ -42,6 +48,7 @@ class NewsCacheManager:
             'filtered_cache': {}  # Cached filtered results
         }
         logger.info(f"NewsCacheManager initialized with {default_ttl}s TTL")
+        self.summarizer = OpenAICompatSummarizer() if OpenAICompatSummarizer else None
 
     def get_news(
         self,
@@ -87,6 +94,9 @@ class NewsCacheManager:
                 # Return cached data if available, even if expired
                 return self.cache[source]['raw_news']
 
+            self._prepare_summaries(new_items)
+            self._apply_ai_summaries(new_items, source=source)
+
             # Update cache
             self._update_cache(source, new_items)
 
@@ -172,7 +182,8 @@ class NewsCacheManager:
             MD5 hash string
         """
         title = self._normalize_text(item.get('title', ''))
-        summary = self._normalize_text(item.get('summary', '')[:200])  # First 200 chars
+        summary_source = item.get('summary_raw', item.get('summary', ''))
+        summary = self._normalize_text(str(summary_source)[:200])  # First 200 chars
 
         # Combine title and summary
         combined = f"{title}|{summary}"
@@ -228,6 +239,19 @@ class NewsCacheManager:
         self.cache[source]['last_fetch'] = datetime.now()
         logger.info(f"📦 Updated cache for {source} with {len(items)} items")
 
+    def _prepare_summaries(self, items: List[Dict]):
+        for item in items:
+            if 'summary_raw' not in item:
+                item['summary_raw'] = item.get('summary', '')
+
+    def _apply_ai_summaries(self, items: List[Dict], source: Optional[str] = None):
+        if not items or not self.summarizer or not getattr(self.summarizer, 'enabled', False):
+            return
+        try:
+            self.summarizer.summarize_items(items, source=source)
+        except Exception as exc:
+            logger.warning(f"AI summarization skipped due to error: {exc}")
+
     def get_filtered_news(
         self,
         source_df: pd.DataFrame,