Force sync entire directory including feature_pipeline.py

feature_pipeline.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import re
 import threading
 from datetime import datetime
@@ -14,6 +15,8 @@ from bs4 import BeautifulSoup
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+logger = logging.getLogger(__name__)
+
 def _browser_headers() -> dict:
     return {
         "User-Agent": (
@@ -28,12 +31,24 @@ def _browser_headers() -> dict:
         "Accept-Language": "en-US,en;q=0.9",
         "Accept-Encoding": "gzip, deflate, br",
         "Connection": "keep-alive",
-        "Upgrade-Insecure-Requests": "1",
         "Cache-Control": "no-cache",
         "Pragma": "no-cache",
         "Referer": "https://www.screener.in/",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
     }
 
+def _clean_number(text: str) -> float:
+    return float(text.replace(",", "").strip())
+
+@dataclass(frozen=True)
+class CapThresholds:
+    large: float = 20000.0
+    mid: float = 5000.0
+
 class _ThreadLocalSessionFactory:
     def __init__(
         self,
@@ -76,7 +91,6 @@ class _ThreadLocalSessionFactory:
         if not hasattr(self._local, "session"):
             self._local.session = self._build_session()
 
-            # Warm up cookies / anti-bot checks by visiting the homepage first.
             if self.prime_homepage:
                 try:
                     self._local.session.get(
@@ -93,6 +107,7 @@ class ScreenerScraper:
         self,
         timeout: Tuple[float, float] = (5.0, 15.0),
         pool_maxsize: int = 32,
+        thresholds: CapThresholds = CapThresholds(),
     ):
         self._session_factory = _ThreadLocalSessionFactory(
             timeout=timeout,
@@ -100,23 +115,16 @@ class ScreenerScraper:
             prime_homepage=True,
         )
         self.timeout = timeout
+        self.thresholds = thresholds
 
     def _session(self) -> requests.Session:
         return self._session_factory.get()
 
     @staticmethod
-    def _make_soup(html: str):
-        try:
-            return BeautifulSoup(html, "lxml")
-        except Exception:
-            return BeautifulSoup(html, "html.parser")
+    def _normalize_symbol(symbol: str) -> str:
+        return quote(symbol.strip().upper(), safe="-._~")
 
-    @staticmethod
-    def _normalize_ticker(ticker: str) -> str:
-        t = ticker.strip().upper()
-        return quote(t, safe="-._~")
-
-    def _fetch_url(self, url: str) -> str:
+    def _fetch(self, url: str) -> str:
         session = self._session()
         response = session.get(url, timeout=self.timeout)
 
@@ -125,48 +133,117 @@ class ScreenerScraper:
 
         preview = ""
         try:
-            preview = response.text[:300].replace("\n", " ").replace("\r", " ")
+            preview = response.text[:250].replace("\n", " ").replace("\r", " ")
         except Exception:
-            preview = "<no response text>"
+            preview = "<no preview>"
 
         raise RuntimeError(
-            f"Request failed.\n"
+            f"Request failed\n"
             f"URL: {url}\n"
             f"Status: {response.status_code}\n"
             f"Reason: {response.reason}\n"
-            f"Response preview: {preview}"
+            f"Preview: {preview}"
         )
 
-    def _fetch_html(self, ticker: str, consolidated: bool = True) -> str:
-        t = self._normalize_ticker(ticker)
+    @staticmethod
+    def _parse_market_cap_crore(html: str) -> Optional[float]:
+        patterns = [
+            r"Mkt Cap:\s*([0-9][0-9,]*(?:\.[0-9]+)?)\s*Crore",
+            r"Market Cap\s*₹\s*([0-9][0-9,]*(?:\.[0-9]+)?)\s*Cr\.?",
+            r"Market Cap:\s*₹\s*([0-9][0-9,]*(?:\.[0-9]+)?)\s*Cr\.?",
+            r"Market Capitalization.*?₹\s*([0-9][0-9,]*(?:\.[0-9]+)?)\s*Cr\.?",
+        ]
+
+        for pattern in patterns:
+            match = re.search(pattern, html, flags=re.IGNORECASE | re.DOTALL)
+            if match:
+                try:
+                    return _clean_number(match.group(1))
+                except Exception:
+                    continue
+
+        return None
 
-        if consolidated:
-            urls = [
-                f"https://www.screener.in/company/{t}/consolidated/",
-                f"https://www.screener.in/company/{t}/",
+    def _fetch_screener_html(self, symbol: str, consolidated: bool = True) -> str:
+        s = self._normalize_symbol(symbol)
+        urls = (
+            [
+                f"https://www.screener.in/company/{s}/consolidated/",
+                f"https://www.screener.in/company/{s}/",
             ]
-        else:
-            urls = [
-                f"https://www.screener.in/company/{t}/",
-                f"https://www.screener.in/company/{t}/consolidated/",
+            if consolidated
+            else [
+                f"https://www.screener.in/company/{s}/",
+                f"https://www.screener.in/company/{s}/consolidated/",
             ]
+        )
 
         last_error: Optional[Exception] = None
-
         for url in urls:
             try:
-                html = self._fetch_url(url)
+                html = self._fetch(url)
                 return html
             except Exception as e:
                 last_error = e
-                continue
 
-        raise RuntimeError(
-            f"Failed to fetch data for '{ticker}'. "
-            f"Last error: {last_error}"
-        ) from last_error
+        raise RuntimeError(f"Failed to fetch Screener page for {symbol}. Last error: {last_error}") from last_error
+
+    def _fetch_nse_quote_html(self, symbol: str) -> str:
+        s = self._normalize_symbol(symbol)
+        url = f"https://www.nseindia.com/get-quotes/equity?symbol={s}"
+        session = self._session()
+        try:
+            session.get("https://www.nseindia.com/", timeout=self.timeout)
+        except requests.RequestException:
+            pass
+        return self._fetch(url)
+
+    def get_market_cap_crore(self, symbol: str, consolidated: bool = True) -> Optional[float]:
+        try:
+            html = self._fetch_screener_html(symbol, consolidated=consolidated)
+            cap = self._parse_market_cap_crore(html)
+            if cap is not None:
+                return cap
+        except Exception as e:
+            logger.warning("Screener lookup failed for %s: %s", symbol, e)
+
+        try:
+            html = self._fetch_nse_quote_html(symbol)
+            cap = self._parse_market_cap_crore(html)
+            if cap is not None:
+                return cap
+        except Exception as e:
+            logger.warning("NSE lookup failed for %s: %s", symbol, e)
+        return None
+
+    def classify_market_cap(self, market_cap_crore: Optional[float]) -> str:
+        if market_cap_crore is None:
+            return "Unknown"
+        if market_cap_crore >= self.thresholds.large:
+            return "Large Cap"
+        if market_cap_crore >= self.thresholds.mid:
+            return "Mid Cap"
+        return "Small Cap"
+
+    def get_cap_info(self, symbol: str, consolidated: bool = True) -> dict:
+        cap = self.get_market_cap_crore(symbol, consolidated=consolidated)
+        return {
+            "symbol": symbol.upper().strip(),
+            "market_cap_crore": cap,
+            "market_cap_class": self.classify_market_cap(cap),
+        }
+
+    # -- Keep original helper methods for get_stock_info --
+    def _fetch_html(self, ticker: str, consolidated: bool = True) -> str:
+        return self._fetch_screener_html(ticker, consolidated)
 
     @staticmethod
+    def _make_soup(html: str):
+        try:
+            return BeautifulSoup(html, "lxml")
+        except Exception:
+            return BeautifulSoup(html, "html.parser")
+    @staticmethod
     def _clean_text(value: str) -> str:
         return " ".join(value.split()).replace("₹", "Rs.")