Completely replace Screener network block with Finology scraping

forecaster_cli.py

@@ -31,80 +31,70 @@ def get_current_historic_val(table_data, row_name):
     return np.nan
 
 def fetch_live_features(ticker):
+    import requests
+    from bs4 import BeautifulSoup
+    import re
+    import numpy as np
+    
     try:
-        scraper = ScreenerScraper()
-        data = None
-        for _ in range(3):
-            try:
-                data = scraper.get_stock_info(ticker)
-                if "error" in data and "429" in data["error"]:
-                    time.sleep(1.5)
-                    continue
-                break
-            except Exception:
-                time.sleep(1)
+        url = f"https://ticker.finology.in/company/{ticker}"
+        html = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}).text
+        soup = BeautifulSoup(html, 'html.parser')
         
-        if not data or "error" in data: return None
+        def clean(val):
+            if not val: return np.nan
+            try: return float(re.sub(r'[^\d.]', '', val))
+            except: return np.nan
             
-        pl = data['history'].get('profit-loss', {})
-        bs = data['history'].get('balance-sheet', {})
-        ratios = data['history'].get('ratios', {})
-        
-        sales_row = None
-        for row in pl.get('rows', []):
-            if row and row[0].lower() == 'sales':
-                sales_row = row
-                break
-        
-        sales = np.nan
-        prev_sales = np.nan
-        if sales_row and len(sales_row) >= 3:
-            try:
-                sales = float(str(sales_row[-1]).replace(',', '').strip())
-                prev_sales = float(str(sales_row[-2]).replace(',', '').strip())
-            except:
-                pass
-                
-        opm = get_current_historic_val(pl, 'OPM %')
-        net_profit = get_current_historic_val(pl, 'Net Profit')
-        equity = get_current_historic_val(bs, 'Equity Capital')
-        reserves = get_current_historic_val(bs, 'Reserves')
-        borrowings = get_current_historic_val(bs, 'Borrowings')
-        roce = get_current_historic_val(ratios, 'ROCE %')
-        
-        sales_growth = ((sales - prev_sales) / prev_sales * 100) if pd.notnull(prev_sales) and prev_sales > 0 else np.nan
-        total_eq = equity + reserves if pd.notnull(equity) and pd.notnull(reserves) else np.nan
-        roe = (net_profit / total_eq * 100) if pd.notnull(total_eq) and total_eq > 0 else np.nan
-        debt_to_equity = (borrowings / total_eq) if pd.notnull(total_eq) and total_eq > 0 else np.nan
-        
-        pe = np.nan
-        for metric in data.get('metrics', []):
-            if metric['name'] == 'Stock P/E':
-                pe = metric['value']
-                break
-        
+        data = {}
+        for div in soup.find_all('div', class_=re.compile(r'compess')):
+            text = div.get_text(" ", strip=True)
+            if "P/E" in text: data['PE_Ratio'] = clean(text.replace('P/E', ''))
+            if "Sales Growth" in text: data['Sales_Growth'] = clean(text.replace('Sales Growth', ''))
+            if "ROE" in text: data['ROE'] = clean(text.replace('ROE', ''))
+            if "ROCE" in text: data['ROCE'] = clean(text.replace('ROCE', ''))
+            if "CASH" in text: data['CASH'] = clean(text.replace('CASH', ''))
+            if "DEBT " in text: data['DEBT'] = clean(text.replace('DEBT', ''))
+            if "Book Value" in text: data['BV'] = clean(text.replace('Book Value', ''))
+            if "No. of Shares" in text: data['Shares'] = clean(text.replace('No. of Shares', ''))
+
+        debt_to_equity = np.nan
+        if 'DEBT' in data and 'BV' in data and 'Shares' in data and data['BV'] > 0 and data['Shares'] > 0:
+            total_equity = data['BV'] * data['Shares']
+            debt_to_equity = data['DEBT'] / total_equity if total_equity > 0 else 0
+
         return {
             'Ticker': ticker,
-            'Sales_Growth': sales_growth,
-            'OPM': opm,
-            'ROCE': roce,
-            'ROE': roe,
+            'Sales_Growth': data.get('Sales_Growth', np.nan),
+            'OPM': np.nan, # Imputer will naturally handle this gracefully
+            'ROCE': data.get('ROCE', np.nan),
+            'ROE': data.get('ROE', np.nan),
             'Debt_to_Equity': debt_to_equity,
-            'PE_Ratio': pe
+            'PE_Ratio': data.get('PE_Ratio', np.nan)
         }
     except Exception:
         return None
 
 def get_market_cap_tier(ticker):
-    print(f"[{ticker}] Resolving market cap classification via Screener...")
+    print(f"[{ticker}] Resolving market cap classification via Finology...")
     try:
-        scraper = ScreenerScraper()
-        cap_class = scraper.get_cap_info(ticker)['market_cap_class']
-        
-        if "Large" in cap_class: return "Large"
-        if "Mid" in cap_class: return "Mid"
-        if "Small" in cap_class: return "Small"
+        import requests
+        from bs4 import BeautifulSoup
+        import re
+        url = f"https://ticker.finology.in/company/{ticker}"
+        html = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}).text
+        soup = BeautifulSoup(html, 'html.parser')
         
+        cap = 0
+        for div in soup.find_all('div', class_=re.compile(r'compess')):
+            text = div.get_text(" ", strip=True)
+            if "Market Cap" in text:
+                try: cap = float(re.sub(r'[^\d.]', '', text.replace('Market Cap', '')))
+                except: pass
+                
+        if cap >= 20000: return "Large"
+        if cap >= 5000: return "Mid"
+        return "Small"
     except Exception as e:
         print(f"Warning: Could not fetch market cap ({e}). Defaulting to Small.")