fix: add Yahoo crumb auth for v10/quoteSummary company info

backend/app/services/data_ingestion/yahoo.py

@@ -50,12 +50,29 @@ _PERIOD_MAP = {
 
 _USE_CURL = False
 _curl_session = None
+_yahoo_crumb = None
 
 try:
     from curl_cffi.requests import Session as CurlSession
     _curl_session = CurlSession(impersonate="chrome")
     _USE_CURL = True
-    logger.info("curl_cffi available — using direct Yahoo API with Chrome TLS")
+
+    # Acquire Yahoo consent cookies + crumb token for API authentication
+    try:
+        # Step 1: Visit fc.yahoo.com to get cookies
+        _curl_session.get("https://fc.yahoo.com", timeout=5)
+        # Step 2: Fetch crumb token
+        crumb_resp = _curl_session.get(
+            "https://query2.finance.yahoo.com/v1/test/getcrumb", timeout=5
+        )
+        if crumb_resp.status_code == 200 and crumb_resp.text:
+            _yahoo_crumb = crumb_resp.text.strip()
+            logger.info("curl_cffi available — using direct Yahoo API with Chrome TLS (crumb acquired)")
+        else:
+            logger.info("curl_cffi available — using direct Yahoo API with Chrome TLS (no crumb)")
+    except Exception as e:
+        logger.warning("Crumb acquisition failed: %s — chart API still works", e)
+
 except ImportError:
     logger.info("curl_cffi not available — using yfinance (local dev mode)")
 
@@ -134,59 +151,62 @@ def _direct_fetch_info(ticker: str) -> dict:
     if not _USE_CURL or not _curl_session:
         return _yfinance_fetch_info(ticker)
 
-    # ── Attempt 1: v10/quoteSummary (full fundamentals) ──────────────
-    try:
-        url = _SUMMARY_URL.format(ticker=ticker)
-        modules = "summaryProfile,summaryDetail,defaultKeyStatistics,financialData,price"
-        resp = _curl_session.get(url, params={"modules": modules}, timeout=10)
-        if resp.status_code == 200:
-            data = resp.json()
-            qr = data.get("quoteSummary", {}).get("result")
-            if qr and len(qr) > 0:
-                r = qr[0]
-                profile = r.get("summaryProfile", {})
-                detail = r.get("summaryDetail", {})
-                stats = r.get("defaultKeyStatistics", {})
-                fin = r.get("financialData", {})
-                price = r.get("price", {})
-
-                def _raw(obj, key):
-                    """Extract raw value from Yahoo's {raw, fmt} format."""
-                    v = obj.get(key, {})
-                    if isinstance(v, dict):
-                        return v.get("raw")
-                    return v
-
-                return {
-                    "longName": price.get("longName"),
-                    "shortName": price.get("shortName") or ticker,
-                    "currency": price.get("currency", "USD"),
-                    "exchange": price.get("exchangeName") or price.get("exchange"),
-                    "sector": profile.get("sector"),
-                    "industry": profile.get("industry"),
-                    "country": profile.get("country"),
-                    "website": profile.get("website"),
-                    "longBusinessSummary": (profile.get("longBusinessSummary") or "")[:500],
-                    "fullTimeEmployees": profile.get("fullTimeEmployees"),
-                    "marketCap": _raw(price, "marketCap"),
-                    "trailingPE": _raw(detail, "trailingPE") or _raw(stats, "trailingPE"),
-                    "forwardPE": _raw(detail, "forwardPE") or _raw(stats, "forwardPE"),
-                    "priceToBook": _raw(detail, "priceToBook") or _raw(stats, "priceToBook"),
-                    "dividendYield": _raw(detail, "dividendYield"),
-                    "trailingEps": _raw(stats, "trailingEps"),
-                    "fiftyTwoWeekHigh": _raw(detail, "fiftyTwoWeekHigh"),
-                    "fiftyTwoWeekLow": _raw(detail, "fiftyTwoWeekLow"),
-                    "averageVolume": _raw(detail, "averageVolume"),
-                    "beta": _raw(detail, "beta") or _raw(stats, "beta3Year"),
-                    "totalRevenue": _raw(fin, "totalRevenue"),
-                    "profitMargins": _raw(fin, "profitMargins"),
-                    "returnOnEquity": _raw(fin, "returnOnEquity"),
-                    "debtToEquity": _raw(fin, "debtToEquity"),
-                    "regularMarketPrice": _raw(price, "regularMarketPrice"),
-                    "previousClose": _raw(detail, "previousClose"),
-                }
-    except Exception as e:
-        logger.debug("v10 quoteSummary failed for %s: %s", ticker, e)
+    # ── Attempt 1: v10/quoteSummary (full fundamentals, needs crumb) ──
+    if _yahoo_crumb:
+        try:
+            url = _SUMMARY_URL.format(ticker=ticker)
+            modules = "summaryProfile,summaryDetail,defaultKeyStatistics,financialData,price"
+            resp = _curl_session.get(
+                url, params={"modules": modules, "crumb": _yahoo_crumb}, timeout=10
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                qr = data.get("quoteSummary", {}).get("result")
+                if qr and len(qr) > 0:
+                    r = qr[0]
+                    profile = r.get("summaryProfile", {})
+                    detail = r.get("summaryDetail", {})
+                    stats = r.get("defaultKeyStatistics", {})
+                    fin = r.get("financialData", {})
+                    price = r.get("price", {})
+
+                    def _raw(obj, key):
+                        """Extract raw value from Yahoo's {raw, fmt} format."""
+                        v = obj.get(key, {})
+                        if isinstance(v, dict):
+                            return v.get("raw")
+                        return v
+
+                    return {
+                        "longName": price.get("longName"),
+                        "shortName": price.get("shortName") or ticker,
+                        "currency": price.get("currency", "USD"),
+                        "exchange": price.get("exchangeName") or price.get("exchange"),
+                        "sector": profile.get("sector"),
+                        "industry": profile.get("industry"),
+                        "country": profile.get("country"),
+                        "website": profile.get("website"),
+                        "longBusinessSummary": (profile.get("longBusinessSummary") or "")[:500],
+                        "fullTimeEmployees": profile.get("fullTimeEmployees"),
+                        "marketCap": _raw(price, "marketCap"),
+                        "trailingPE": _raw(detail, "trailingPE") or _raw(stats, "trailingPE"),
+                        "forwardPE": _raw(detail, "forwardPE") or _raw(stats, "forwardPE"),
+                        "priceToBook": _raw(detail, "priceToBook") or _raw(stats, "priceToBook"),
+                        "dividendYield": _raw(detail, "dividendYield"),
+                        "trailingEps": _raw(stats, "trailingEps"),
+                        "fiftyTwoWeekHigh": _raw(detail, "fiftyTwoWeekHigh"),
+                        "fiftyTwoWeekLow": _raw(detail, "fiftyTwoWeekLow"),
+                        "averageVolume": _raw(detail, "averageVolume"),
+                        "beta": _raw(detail, "beta") or _raw(stats, "beta3Year"),
+                        "totalRevenue": _raw(fin, "totalRevenue"),
+                        "profitMargins": _raw(fin, "profitMargins"),
+                        "returnOnEquity": _raw(fin, "returnOnEquity"),
+                        "debtToEquity": _raw(fin, "debtToEquity"),
+                        "regularMarketPrice": _raw(price, "regularMarketPrice"),
+                        "previousClose": _raw(detail, "previousClose"),
+                    }
+        except Exception as e:
+            logger.debug("v10 quoteSummary failed for %s: %s", ticker, e)
 
     # ── Attempt 2: chart API meta (basic fallback) ──────────────────
     try: