fix: use curl_cffi for Chrome TLS fingerprint impersonation on cloud

backend/app/services/data_ingestion/yahoo.py

@@ -18,7 +18,6 @@ from functools import partial
 from typing import Any, Dict, List, Optional
 
 import pandas as pd
-import requests
 import yfinance as yf
 
 from app.redis_client import cache_get, cache_set
@@ -37,26 +36,31 @@ _yf_pool = ThreadPoolExecutor(max_workers=4, thread_name_prefix="yfinance")
 YF_TIMEOUT = 15
 
 
-def _create_yf_session() -> requests.Session:
-    """Create a requests session with browser-like headers.
+def _create_yf_session():
+    """Create a session with browser TLS fingerprint impersonation.
 
-    Yahoo Finance blocks bare requests from cloud/datacenter IPs.
-    Using browser headers and proper cookie handling bypasses this.
+    Yahoo Finance blocks requests based on TLS fingerprinting, not just
+    HTTP headers. curl_cffi impersonates Chrome's actual TLS handshake,
+    which is undetectable by Yahoo's anti-bot systems.
+    Falls back to a regular requests.Session for local development.
     """
-    session = requests.Session()
-    session.headers.update({
-        "User-Agent": (
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-            "AppleWebKit/537.36 (KHTML, like Gecko) "
-            "Chrome/131.0.0.0 Safari/537.36"
-        ),
-        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.5",
-        "Accept-Encoding": "gzip, deflate",
-        "Connection": "keep-alive",
-        "Upgrade-Insecure-Requests": "1",
-    })
-    return session
+    try:
+        from curl_cffi.requests import Session
+        session = Session(impersonate="chrome")
+        logger.info("Using curl_cffi session with Chrome TLS impersonation")
+        return session
+    except ImportError:
+        import requests
+        session = requests.Session()
+        session.headers.update({
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/131.0.0.0 Safari/537.36"
+            ),
+        })
+        logger.info("curl_cffi not available, using requests.Session (local dev mode)")
+        return session
 
 
 # Shared session for all yfinance calls

backend/requirements.txt

@@ -36,6 +36,7 @@ hmmlearn==0.3.3
 # HTTP Client
 httpx==0.28.1
 aiohttp==3.11.11
+curl_cffi>=0.7.0
 
 # Utilities
 python-dateutil==2.9.0