Upload 11 files

app.py

@@ -6,7 +6,7 @@ from datetime import datetime
 from data.scraper import NewsScraper
 from engine.analytics import AnalyticsEngine
 
-scraper = NewsScraper()
+scraper = NewsScraper(limit=600)
 engine = AnalyticsEngine()
 
 # Simple persistent cache
@@ -17,32 +17,33 @@ async def run_pipeline(ticker, date_str, progress=gr.Progress()):
     print(f"\n--- Request: {cache_key} ---")
     
     if cache_key in persistent_cache:
-        progress(1.0, "Result loaded from Pulse Memory.")
+        progress(1.0, "Loaded from cache.")
         return persistent_cache[cache_key]
 
     start_total = time.time()
     try:
-        # Validate inputs
-        if not ticker: return "Error: No ticker provided", pd.DataFrame()
+        if not ticker:
+            return "Error: No ticker provided", pd.DataFrame()
         
-        # 0. Initialize
-        progress(0.01, "Neural Cluster: Online")
-        await asyncio.sleep(0.2) # Async sync for stream stability
+        # 0. Send estimated time immediately
+        estimated_seconds = engine.estimate_time(scraper.limit)
+        progress(0.01, f"ETA:{estimated_seconds}s | Initializing...")
+        await asyncio.sleep(0.2)
         
         try:
-            # Parse date but strip time/tz for the scraper to be more lenient
             dt = datetime.strptime(date_str, "%Y-%m-%d")
         except Exception as e:
             return f"Error: Invalid Date Format ({str(e)})", pd.DataFrame()
 
-        # Status: Cleaning
         scraper.cleanup()
         
-        # 1. Scrape Phase
-        progress(0.05, "Pulse Engine: Starting Aggressive Harvesting...")
+        # 1. Scrape Phase (0.02 → 0.30)
+        progress(0.02, f"ETA:{estimated_seconds}s | Collecting headlines...")
         s_start = time.time()
-        # Scale scraper progress to 0.05 -> 0.45
-        articles = await scraper.scrape(ticker, dt, progress_cb=lambda val, msg: progress(0.05 + (val * 0.4), f"Harvest: {msg}"))
+        articles = await scraper.scrape(
+            ticker, dt,
+            progress_cb=lambda val, msg: progress(0.02 + (val * 0.28), f"ETA:{estimated_seconds}s | {msg}")
+        )
         s_time = time.time() - s_start
         
         if not articles: 
@@ -50,52 +51,71 @@ async def run_pipeline(ticker, date_str, progress=gr.Progress()):
             
         df = pd.DataFrame(articles)
         total_analyzed = len(df)
+
+        # Recalculate ETA now that we know article count
+        estimated_seconds = engine.estimate_time(total_analyzed)
+        remaining = max(0, estimated_seconds - int(time.time() - start_total))
         
-        # 2. Analyze Phase
-        progress(0.5, f"Neural Phase: Processing {total_analyzed} headlines...")
+        # 2. Analyze Phase (0.30 → 0.90)
+        progress(0.30, f"ETA:{remaining}s | Running sentiment models on {total_analyzed} articles...")
         a_start = time.time()
-        # Wrap blocking analysis in a thread to keep progress bar responsive
-        df = await asyncio.to_thread(engine.analyze, df, ticker, progress_cb=lambda val, msg: progress(0.5 + (val * 0.4), msg))
-        summary = engine.get_summary(df)
+        
+        def run_analysis():
+            return engine.analyze(
+                df, ticker,
+                progress_cb=lambda val, msg: progress(0.30 + (val * 0.60), f"ETA:{remaining}s | {msg}")
+            )
+        
+        analyzed_df = await asyncio.to_thread(run_analysis)
+        summary = engine.get_summary(analyzed_df)
         a_time = time.time() - a_start
         
         total_time = time.time() - start_total
         
         # 3. Format Output
-        progress(0.95, "Finalizing Pulse Report...")
-        report_text = f"🚀 {ticker} PULSE AI INTELLIGENCE\n"
+        progress(0.95, "Generating report...")
+        
+        report_text = f"{ticker} SENTIMENT REPORT\n"
         report_text += f"{'='*30}\n"
         report_text += f"PHASE STATS:\n"
-        report_text += f"  > Data Harvest : {s_time:.2f}s\n"
-        report_text += f"  > Neural Compute: {a_time:.2f}s\n"
-        report_text += f"  > TOTAL TIME  : {total_time:.2f}s\n"
-        report_text += f"  > ANALYZED    : {total_analyzed} articles\n"
+        report_text += f"  > Scraping     : {s_time:.2f}s\n"
+        report_text += f"  > ML Analysis  : {a_time:.2f}s\n"
+        report_text += f"  > TOTAL TIME   : {total_time:.2f}s\n"
+        report_text += f"  > ANALYZED     : {total_analyzed} articles\n"
         report_text += f"{'='*30}\n\n"
         report_text += f"QUICK VIBE: {summary['vibe']}/10\n"
-        report_text += f"AVG POLARITY: {summary['avg_polarity']:.3f}\n\n"
+        report_text += f"AVG POLARITY: {summary['avg_polarity']:.3f}\n"
+        report_text += f"DIRECTION RATIO: {summary['dir_ratio']:.3f}\n"
+        report_text += f"CONVICTION SCORE: {summary['conviction_weighted']:.3f}\n"
+        report_text += f"MODEL AGREEMENT: {summary['agreement_rate']:.1%}\n"
+        report_text += f"MOMENTUM TREND: {summary['momentum_delta']:+.3f}\n"
+        report_text += f"COMPOSITE SCORE: {summary['composite_score']:.3f}\n"
+        report_text += f"TAIL RISK: {summary['tail_risk']:.1%}\n\n"
         report_text += f"🔥 HEAVY HITTER SIGNALS:\n"
         for a in summary['heavy_hitters']:
             report_text += f"- {a['title']}\n"
             
-        result = (report_text, df.head(50)) 
+        result = (report_text, analyzed_df.head(50))
         persistent_cache[cache_key] = result
-        progress(1.0, "Intelligence Generated.")
+        progress(1.0, "Done.")
         return result
 
     except Exception as e:
-        print(f"Pipeline Crash: {str(e)}")
-        return f"CRITICAL ERROR: {str(e)}", pd.DataFrame()
+        print(f"Pipeline Error: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return f"ERROR: {str(e)}", pd.DataFrame()
 
 def demo():
-    with gr.Blocks(title="Pulse AI Intelligence") as app:
-        gr.Markdown("# 🚀 Pulse AI: Market Intelligence")
+    with gr.Blocks(title="Sentiment Analyzer") as app:
+        gr.Markdown("# Sentiment Analyzer")
         with gr.Row():
             ticker = gr.Textbox(label="Ticker Symbol", value="TSLA")
             date = gr.Textbox(label="Lookback Date (YYYY-MM-DD)", value="2024-01-01")
         
-        btn = gr.Button("Fetch Intelligence")
-        output = gr.Textbox(label="Pulse Report Summary")
-        table = gr.Dataframe(label="Neural Dataset")
+        btn = gr.Button("Analyze Sentiment")
+        output = gr.Textbox(label="Report")
+        table = gr.Dataframe(label="Dataset")
         
         btn.click(
             fn=run_pipeline, 

data/scraper.py

@@ -1,8 +1,8 @@
 import asyncio
 import aiohttp
 import xml.etree.ElementTree as ET
-import sys
 import ssl
+import re
 from datetime import datetime
 from email.utils import parsedate_to_datetime
 import glob
@@ -17,17 +17,18 @@ class NewsScraper:
 
     async def fetch_feed(self, session, url):
         try:
-            async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as response:
+            async with session.get(url, timeout=aiohttp.ClientTimeout(total=8)) as response:
                 if response.status == 200:
                     return await response.text()
-        except: pass
+        except:
+            pass
         return ""
 
     def parse_feed(self, xml_text, lookback_date):
         articles = []
-        if not xml_text: return articles
+        if not xml_text:
+            return articles
         try:
-            # RSS Dates are often UTC, so we compare dates (year/month/day) to be safe
             lb_date = lookback_date.date()
             root = ET.fromstring(xml_text)
             for item in root.findall('.//item'):
@@ -37,55 +38,139 @@ class NewsScraper:
                 if title and link and pub_date_str:
                     try:
                         pub_dt = parsedate_to_datetime(pub_date_str)
-                        # More lenient: if it's the same day or later, we keep it
                         if pub_dt.date() >= lb_date:
                             articles.append({
-                                'title': title, 'link': link, 
+                                'title': title, 'link': link,
                                 'pub_date': pub_date_str, 'timestamp': pub_dt.isoformat()
                             })
-                    except: pass
-        except: pass
+                    except:
+                        pass
+        except:
+            pass
         return articles
 
-    async def scrape(self, ticker, lookback_date, progress_cb=None):
-        # Even more aggressive query set
-        queries = [
-            ticker, f"{ticker} stock", f"{ticker} news", f"{ticker} market", 
-            f"{ticker} earnings", f"{ticker} analyst", f"{ticker} forecast",
-            f"{ticker} price target", f"{ticker} institutional", f"{ticker} hedge fund",
-            f"{ticker} options", f"{ticker} technical", f"{ticker} dividend",
-            f"{ticker} industry", f"{ticker} competitor", f"{ticker} share price"
+    def _build_queries(self, ticker):
+        """Generate a massive, diverse set of search queries to maximize article yield."""
+        t = ticker
+        base = [
+            t, f"{t} stock", f"{t} news", f"{t} market", f"{t} earnings",
+            f"{t} analyst", f"{t} forecast", f"{t} price target",
+            f"{t} options", f"{t} technical", f"{t} dividend",
+            f"{t} industry", f"{t} competitor", f"{t} share price",
+            f"{t} hedge fund", f"{t} institutional",
+        ]
+        
+        # Financial action queries
+        actions = [
+            f"{t} buy sell hold", f"{t} upgrade downgrade", f"{t} outperform underperform",
+            f"{t} bullish bearish", f"{t} momentum", f"{t} breakout breakdown",
+            f"{t} rally crash", f"{t} surge plunge", f"{t} soar tumble",
+            f"{t} gains losses", f"{t} beat miss expectations",
+        ]
+        
+        # Corporate event queries
+        events = [
+            f"{t} CEO news", f"{t} quarterly results", f"{t} revenue profit",
+            f"{t} guidance outlook", f"{t} acquisition merger",
+            f"{t} lawsuit legal SEC", f"{t} insider trading",
+            f"{t} IPO offering", f"{t} buyback repurchase",
+            f"{t} partnership deal", f"{t} product launch",
+            f"{t} layoffs restructuring", f"{t} expansion growth",
+        ]
+        
+        # Analyst and research queries
+        research = [
+            f"{t} wall street", f"{t} Goldman Sachs", f"{t} Morgan Stanley",
+            f"{t} JP Morgan", f"{t} analyst rating", f"{t} price prediction",
+            f"{t} short interest", f"{t} short squeeze",
+            f"{t} put call ratio", f"{t} unusual activity",
+            f"{t} fund holdings", f"{t} 13F filing",
+        ]
+        
+        # Sector and macro queries
+        macro = [
+            f"{t} sector outlook", f"{t} industry trend", f"{t} supply chain",
+            f"{t} regulation policy", f"{t} inflation impact",
+            f"{t} interest rate", f"{t} trade war tariff",
+            f"{t} innovation technology", f"{t} ESG sustainability",
         ]
         
+        # Time-sensitive queries
+        time_q = [
+            f"{t} today", f"{t} this week", f"{t} latest",
+            f"{t} breaking news", f"{t} update",
+            f"{t} premarket", f"{t} after hours",
+        ]
+        
+        all_queries = base + actions + events + research + macro + time_q
+        return all_queries
+
+    async def scrape(self, ticker, lookback_date, progress_cb=None):
+        queries = self._build_queries(ticker)
+        total_queries = len(queries)
+        
         all_articles = []
         seen = set()
         
-        connector = aiohttp.TCPConnector(limit=100, ssl=self.ssl_context)
+        # Batch fetch: fire all requests concurrently for speed
+        connector = aiohttp.TCPConnector(limit=50, ssl=self.ssl_context)
         async with aiohttp.ClientSession(connector=connector) as session:
-            for i, q in enumerate(queries):
-                if len(all_articles) >= self.limit: break
+            # Build all URLs
+            urls = []
+            for q in queries:
+                encoded = q.replace(' ', '+')
+                urls.append((q, f"https://news.google.com/rss/search?q={encoded}&hl=en-US&gl=US&ceid=US:en"))
+            
+            # Also add general financial news feeds to pad the count
+            general_feeds = [
+                "https://news.google.com/rss/headlines/section/topic/BUSINESS?hl=en-US&gl=US&ceid=US:en",
+                "https://news.google.com/rss/search?q=stock+market&hl=en-US&gl=US&ceid=US:en",
+                "https://news.google.com/rss/search?q=wall+street+today&hl=en-US&gl=US&ceid=US:en",
+                "https://news.google.com/rss/search?q=stocks+trading&hl=en-US&gl=US&ceid=US:en",
+                "https://news.google.com/rss/search?q=financial+markets&hl=en-US&gl=US&ceid=US:en",
+            ]
+            for gf in general_feeds:
+                urls.append(("General Market", gf))
+            
+            # Fire all requests concurrently in batches of 20
+            batch_size = 20
+            for batch_start in range(0, len(urls), batch_size):
+                if len(all_articles) >= self.limit:
+                    break
+                    
+                batch = urls[batch_start:batch_start + batch_size]
+                tasks = [self.fetch_feed(session, url) for _, url in batch]
+                results = await asyncio.gather(*tasks, return_exceptions=True)
                 
-                # Signal activity even if no results found yet
-                if progress_cb:
-                    progress_cb(len(all_articles) / self.limit, f"Searching: '{q}' ({len(all_articles)} found)")
-                
-                url = f"https://news.google.com/rss/search?q={q.replace(' ', '+')}&hl=en-US&gl=US&ceid=US:en"
-                xml = await self.fetch_feed(session, url)
-                res = self.parse_feed(xml, lookback_date)
+                for (query_name, _), xml in zip(batch, results):
+                    if isinstance(xml, Exception) or not xml:
+                        continue
+                    parsed = self.parse_feed(xml, lookback_date)
+                    for a in parsed:
+                        if a['link'] not in seen:
+                            seen.add(a['link'])
+                            all_articles.append(a)
+                            if len(all_articles) >= self.limit:
+                                break
                 
-                for a in res:
-                    if a['link'] not in seen:
-                        seen.add(a['link'])
-                        all_articles.append(a)
-                        # Update progress frequently
-                        if len(all_articles) % 10 == 0 and progress_cb:
-                            progress_cb(len(all_articles) / self.limit, f"Harvested {len(all_articles)}/{self.limit} articles...")
-                        if len(all_articles) >= self.limit: break
+                # Report progress
+                if progress_cb:
+                    scrape_progress = min(len(all_articles) / self.limit, 1.0)
+                    progress_cb(
+                        scrape_progress,
+                        f"Collecting headlines: {len(all_articles)}/{self.limit}"
+                    )
                 
+                # Small delay between batches to avoid rate limiting
+                await asyncio.sleep(0.1)
+        
+        print(f"[Scraper] Total unique articles collected: {len(all_articles)}")
         return all_articles[:self.limit]
 
     @staticmethod
     def cleanup():
         for f in glob.glob("*.csv"):
-            try: os.remove(f)
-            except: pass
+            try:
+                os.remove(f)
+            except:
+                pass

engine/analytics.py

@@ -3,42 +3,232 @@ import numpy as np
 import torch
 from transformers import pipeline
 from sentence_transformers import CrossEncoder
+from scipy import stats as scipy_stats
 
 class AnalyticsEngine:
     def __init__(self):
         self.device = 0 if torch.cuda.is_available() else -1
-        self.sentiment_pipe = pipeline("sentiment-analysis", model="ProsusAI/finbert", device=self.device)
-        self.ranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", device='cuda' if torch.cuda.is_available() else 'cpu')
+        torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+        # Model 1: FinBERT (Financial domain sentiment)
+        self.finbert = pipeline(
+            "sentiment-analysis",
+            model="ProsusAI/finbert",
+            device=self.device,
+            max_length=512,
+            truncation=True
+        )
+
+        # Model 2: DistilRoBERTa (General sentiment, trained on diverse data)
+        self.distilroberta = pipeline(
+            "sentiment-analysis",
+            model="distilbert/distilbert-base-uncased-finetuned-sst-2-english",
+            device=self.device,
+            max_length=512,
+            truncation=True
+        )
+
+        # Model 3: Cross-Encoder for significance ranking
+        self.ranker = CrossEncoder(
+            "cross-encoder/ms-marco-MiniLM-L-6-v2",
+            device=torch_device
+        )
+
+    def _map_finbert(self, label, score):
+        """FinBERT: positive/negative/neutral → polarity."""
+        mapping = {'positive': 1.0, 'neutral': 0.0, 'negative': -1.0}
+        return mapping.get(label, 0.0) * score
+
+    def _map_distilroberta(self, label, score):
+        """DistilRoBERTa: POSITIVE/NEGATIVE → polarity."""
+        if label == "POSITIVE":
+            return score
+        elif label == "NEGATIVE":
+            return -score
+        return 0.0
 
     def analyze(self, df, ticker, progress_cb=None):
         titles = df['title'].tolist()
         total = len(titles)
-        
-        # 1. Sentiment Phase
-        if progress_cb: progress_cb(0.1, "Neural Phase: Analyzing Sentiment Polarity...")
-        results = self.sentiment_pipe(titles, batch_size=32)
-        res_df = pd.DataFrame(results)
-        df = pd.concat([df, res_df], axis=1)
-        
-        # 2. Quant Features
-        if progress_cb: progress_cb(0.5, "Math Phase: Calculating Quant Features...")
-        mapping = {'positive': 1.0, 'neutral': 0.0, 'negative': -1.0}
-        df['pol'] = df['label'].map(mapping)
-        df['conviction'] = df['score'] * df['pol'].abs()
-        
-        # 3. Significance Phase
-        if progress_cb: progress_cb(0.7, "Insight Phase: Ranking Heavy Hitters...")
-        query = f"Market moving news for {ticker}"
+        batch_size = 32
+
+        # ──────────────────────────────────────────────
+        # Phase 1: FinBERT Sentiment (financial domain)
+        # ──────────────────────────────────────────────
+        if progress_cb:
+            progress_cb(0.05, f"Model 1/2: FinBERT analyzing {total} headlines...")
+
+        finbert_results = []
+        for i in range(0, total, batch_size):
+            batch = titles[i:i + batch_size]
+            finbert_results.extend(self.finbert(batch))
+            if progress_cb:
+                progress_cb(0.05 + (i / total) * 0.25, f"FinBERT: {min(i + batch_size, total)}/{total}")
+
+        df['finbert_label'] = [r['label'] for r in finbert_results]
+        df['finbert_score'] = [r['score'] for r in finbert_results]
+        df['finbert_pol'] = df.apply(
+            lambda row: self._map_finbert(row['finbert_label'], row['finbert_score']), axis=1
+        )
+
+        # ──────────────────────────────────────────────
+        # Phase 2: DistilRoBERTa Sentiment (general)
+        # ──────────────────────────────────────────────
+        if progress_cb:
+            progress_cb(0.35, f"Model 2/2: DistilRoBERTa analyzing {total} headlines...")
+
+        roberta_results = []
+        for i in range(0, total, batch_size):
+            batch = titles[i:i + batch_size]
+            roberta_results.extend(self.distilroberta(batch))
+            if progress_cb:
+                progress_cb(0.35 + (i / total) * 0.25, f"DistilRoBERTa: {min(i + batch_size, total)}/{total}")
+
+        df['roberta_label'] = [r['label'] for r in roberta_results]
+        df['roberta_score'] = [r['score'] for r in roberta_results]
+        df['roberta_pol'] = df.apply(
+            lambda row: self._map_distilroberta(row['roberta_label'], row['roberta_score']), axis=1
+        )
+
+        # ──────────────��───────────────────────────────
+        # Phase 3: Ensemble Fusion
+        # ──────────────────────────────────────────────
+        if progress_cb:
+            progress_cb(0.65, "Computing ensemble sentiment fusion...")
+
+        # Weighted ensemble: FinBERT gets 0.6 weight (domain expert), RoBERTa gets 0.4
+        FINBERT_WEIGHT = 0.6
+        ROBERTA_WEIGHT = 0.4
+        df['ensemble_pol'] = (df['finbert_pol'] * FINBERT_WEIGHT) + (df['roberta_pol'] * ROBERTA_WEIGHT)
+
+        # Conviction: how confident BOTH models are (geometric mean of confidences)
+        df['conviction'] = np.sqrt(df['finbert_score'] * df['roberta_score']) * df['ensemble_pol'].abs()
+
+        # Agreement score: do both models agree on direction?
+        df['agreement'] = (np.sign(df['finbert_pol']) == np.sign(df['roberta_pol'])).astype(float)
+
+        # Statistical features
+        df['z_score'] = scipy_stats.zscore(df['ensemble_pol'], nan_policy='omit')
+        df['momentum'] = df['ensemble_pol'].rolling(window=max(10, total // 20), min_periods=1).mean()
+
+        # For backward compat with the rest of the pipeline
+        df['label'] = df['finbert_label']
+        df['score'] = df['finbert_score']
+        df['pol'] = df['ensemble_pol']
+
+        # ──────────────────────────────────────────────
+        # Phase 4: Significance Ranking
+        # ──────────────────────────────────────────────
+        if progress_cb:
+            progress_cb(0.75, "Ranking headline significance...")
+
+        query = f"Major market moving news for {ticker} stock"
         pairs = [[query, t] for t in titles]
-        df['significance'] = self.ranker.predict(pairs, batch_size=32)
-        
-        if progress_cb: progress_cb(1.0, "Analysis Complete.")
+        df['significance'] = self.ranker.predict(pairs, batch_size=batch_size)
+
+        if progress_cb:
+            progress_cb(1.0, "Analysis complete.")
         return df
 
     def get_summary(self, df):
+        """
+        Advanced scoring that avoids the neutral trap.
+
+        The old formula: vibe = ((mean_pol + 1) * 4.5) + conviction
+        Problem: mean_pol ~ 0 for most tickers → vibe ~ 4.5 always.
+
+        New approach: Multi-signal composite score using:
+        1. Ensemble polarity (weighted mean of 2 models)
+        2. Directional ratio (what % of articles are positive vs negative)
+        3. Conviction-weighted polarity (strong signals count more)
+        4. Agreement factor (when both models agree, amplify the signal)
+        5. Momentum trend (is sentiment accelerating?)
+        """
+        n = len(df)
+
+        # Signal 1: Raw ensemble polarity [-1, 1]
+        mean_pol = df['ensemble_pol'].mean()
+
+        # Signal 2: Directional ratio [-1, 1]
+        # Instead of treating neutral as 0, count the RATIO of positive to negative
+        pos_count = (df['ensemble_pol'] > 0.1).sum()
+        neg_count = (df['ensemble_pol'] < -0.1).sum()
+        total_directional = pos_count + neg_count
+        if total_directional > 0:
+            dir_ratio = (pos_count - neg_count) / total_directional
+        else:
+            dir_ratio = 0.0
+
+        # Signal 3: Conviction-weighted polarity [-1, 1]
+        if df['conviction'].sum() > 0:
+            conv_weighted = (df['ensemble_pol'] * df['conviction']).sum() / df['conviction'].sum()
+        else:
+            conv_weighted = 0.0
+
+        # Signal 4: Agreement-amplified signal
+        agreed = df[df['agreement'] == 1.0]
+        if len(agreed) > 0:
+            agreed_pol = agreed['ensemble_pol'].mean()
+        else:
+            agreed_pol = mean_pol
+
+        # Signal 5: Momentum (is sentiment trending?)
+        if len(df) >= 10:
+            recent = df['ensemble_pol'].tail(n // 3).mean()
+            older = df['ensemble_pol'].head(n // 3).mean()
+            momentum_delta = recent - older
+        else:
+            momentum_delta = 0.0
+
+        # Composite score: weighted combination
+        composite = (
+            mean_pol * 0.20 +          # Raw polarity
+            dir_ratio * 0.25 +          # Directional strength
+            conv_weighted * 0.25 +      # Conviction-weighted
+            agreed_pol * 0.20 +         # Agreement signal
+            momentum_delta * 0.10       # Momentum trend
+        )
+
+        # Map composite [-1, 1] → vibe [1, 10]
+        # Using a sigmoid-like scaling to push away from center
+        stretched = np.sign(composite) * (abs(composite) ** 0.7)
+        vibe = int(np.clip(round((stretched + 1) * 4.5 + 0.5), 1, 10))
+
+        # Additional quant metrics for the report
+        avg_conviction = df['conviction'].mean()
+        tail_risk = (df['ensemble_pol'] < -0.5).sum() / n
+        entropy = df['ensemble_pol'].value_counts(normalize=True).std()
+        quant_confidence = avg_conviction * (1 - entropy) if entropy < 1 else avg_conviction
+
         summary = {
-            "avg_polarity": df['pol'].mean(),
-            "vibe": int(np.clip(((df['pol'].mean() + 1) * 4.5) + (df['conviction'].mean()), 1, 10)),
-            "heavy_hitters": df.sort_values(by='significance', ascending=False).head(5).to_dict('records')
+            "avg_polarity": float(mean_pol),
+            "vibe": vibe,
+            "dir_ratio": float(dir_ratio),
+            "conviction_weighted": float(conv_weighted),
+            "agreement_rate": float(df['agreement'].mean()),
+            "momentum_delta": float(momentum_delta),
+            "composite_score": float(composite),
+            "avg_conviction": float(avg_conviction),
+            "tail_risk": float(tail_risk),
+            "quant_confidence": float(quant_confidence),
+            "heavy_hitters": df.sort_values(by='significance', ascending=False).head(8).to_dict('records')
         }
         return summary
+
+    def estimate_time(self, article_count):
+        """
+        Estimate processing time based on article count.
+        Benchmarks (approximate, on HF Spaces free tier):
+        - FinBERT: ~0.15s per batch of 32
+        - DistilRoBERTa: ~0.10s per batch of 32
+        - CrossEncoder: ~0.20s per batch of 32
+        - Scraping: ~8-15s for 600 articles
+        """
+        batches = (article_count + 31) // 32
+        scrape_time = 12  # average scrape time
+        finbert_time = batches * 0.15
+        roberta_time = batches * 0.10
+        ranker_time = batches * 0.20
+        overhead = 3
+        total = scrape_time + finbert_time + roberta_time + ranker_time + overhead
+        return round(total)

requirements.txt

@@ -6,4 +6,5 @@ gradio==4.44.1
 huggingface_hub==0.24.7
 sentence-transformers
 aiohttp
+scipy
 pytest