Update app.py

app.py

@@ -36,30 +36,61 @@ SYSTEM_STATUS = {
 }
 UPDATE_LOCK = threading.Lock()
 
-# Manual mapping for Index Spot symbols
-INDEX_MAP = {
-    "NIFTY": "NSE:NIFTY50-INDEX",
-    "BANKNIFTY": "NSE:NIFTYBANK-INDEX",
-    "FINNIFTY": "NSE:FINNIFTY-INDEX",
-    "MIDCPNIFTY": "NSE:MIDCPNIFTY-INDEX",
-    "SENSEX": "BSE:SENSEX-INDEX",
-    "BANKEX": "BSE:BANKEX-INDEX",
-    "SENSEX50": "BSE:SENSEX50-INDEX"
-}
-
-def get_root_symbol(desc: str, symbol: str) -> str:
-    """Extracts root symbol (e.g. 'RELIANCE', 'SENSEX') from description."""
+# Global Cache for Indices/Stocks Spot Mapping
+SPOT_MAPPING = {} 
+
+def load_spot_mapping(headers):
+    """
+    Downloads Capital Market (CM) files to map Descriptions to exact Spot Symbols.
+    This replaces hardcoded INDEX_MAP.
+    """
+    global SPOT_MAPPING
+    temp_map = {}
+    
     try:
-        parts = desc.strip().split(' ')
-        root = parts[0].upper()
-        return root.replace(':', '').replace('-', '')
-    except:
-        if ':' in symbol:
-            return symbol.split(':')[1]
-        return symbol
+        # 1. NSE CM (Indices & Equities)
+        # We need this to find 'NSE:NIFTY50-INDEX' from description 'Nifty 50'
+        url_nse = "https://public.fyers.in/sym_details/NSE_CM.csv"
+        df_nse = pd.read_csv(
+            url_nse, usecols=[1, 9, 13], 
+            names=['Desc', 'Symbol', 'Inst'], 
+            header=0, on_bad_lines='skip', storage_options=headers
+        )
+        
+        # 2. BSE CM (Sensex etc)
+        url_bse = "https://public.fyers.in/sym_details/BSE_CM.csv"
+        df_bse = pd.read_csv(
+            url_bse, usecols=[1, 9, 13], 
+            names=['Desc', 'Symbol', 'Inst'], 
+            header=0, on_bad_lines='skip', storage_options=headers
+        )
+        
+        # Combine and Process
+        df_all = pd.concat([df_nse, df_bse], ignore_index=True)
+        
+        for _, row in df_all.iterrows():
+            desc = str(row['Desc']).strip().upper()
+            sym = str(row['Symbol']).strip()
+            
+            # Key Logic: Map the Description to the Symbol
+            # This handles 'NIFTY 50' -> 'NSE:NIFTY50-INDEX'
+            # And 'RELIANCE INDUSTRIES LTD' -> 'NSE:RELIANCE-EQ'
+            if desc not in temp_map:
+                temp_map[desc] = sym
+                
+            # Also map the short cleaning name for Indices (e.g., "NIFTY 50")
+            # Fyers FO Desc often matches the start of CM Desc
+            if "-INDEX" in sym:
+                temp_map[desc] = sym
+                
+        SPOT_MAPPING = temp_map
+        print(f">>> [BG TASK] Spot Mapping Loaded: {len(SPOT_MAPPING)} symbols")
+        
+    except Exception as e:
+        print(f"Error loading CM files: {e}")
 
 def update_master_db():
-    global SYSTEM_STATUS, MASTER_DB, SEARCH_INDEX
+    global SYSTEM_STATUS, MASTER_DB, SEARCH_INDEX, SPOT_MAPPING
     
     if not UPDATE_LOCK.acquire(blocking=False):
         return
@@ -70,6 +101,10 @@ def update_master_db():
         
         headers = { "User-Agent": "Mozilla/5.0" }
         
+        # Phase 1: Load Spot/Index Definitions (CM Files)
+        load_spot_mapping(headers)
+
+        # Phase 2: Load Derivatives (FO Files)
         # 1. NSE F&O
         url_nse = "https://public.fyers.in/sym_details/NSE_FO.csv"
         df_nse = pd.read_csv(
@@ -105,25 +140,95 @@ def update_master_db():
         
         temp_db = {}
         
+        # Iterate over derivatives to group them
         for _, row in df.iterrows():
-            sym = row['Symbol']
-            desc = row['Desc']
+            sym = row['Symbol']     # e.g., NSE:NIFTY24JANFUT
+            desc = row['Desc']      # e.g., NIFTY 50 24 JAN FUT
             exch = row['Exch']
             
-            root = get_root_symbol(desc, sym)
+            # --- LOGIC TO FIND ROOT EXACTLY AS IT IS ---
             
-            if root not in temp_db:
-                spot_sym = ""
-                if root in INDEX_MAP:
-                    spot_sym = INDEX_MAP[root]
-                elif exch == "MCX":
-                    spot_sym = "MCX" 
-                elif exch == "BSE":
-                    spot_sym = f"BSE:{root}"
-                else:
-                    spot_sym = f"NSE:{root}-EQ"
+            # 1. Try to find underlying based on Description
+            # Fyers FO Desc: "NIFTY 50 24 JAN..."
+            # We check if "NIFTY 50" exists in our SPOT_MAPPING
+            root_spot = ""
+            desc_upper = desc.strip().upper()
+            
+            # Fast Check: Check common index names first or substring match
+            # This dynamic lookup replaces the INDEX_MAP
+            matched_spot = None
+            
+            # Optimization: Check valid index names
+            # Extract potential name parts (e.g. "NIFTY 50" from "NIFTY 50 24...")
+            desc_parts = desc_upper.split(' ')
+            
+            # Try 1 word, 2 words, 3 words combinations to find match in SPOT_MAPPING
+            # e.g. "NIFTY", "NIFTY 50"
+            for i in range(1, 4):
+                if i > len(desc_parts): break
+                check_name = " ".join(desc_parts[:i])
+                if check_name in SPOT_MAPPING:
+                    matched_spot = SPOT_MAPPING[check_name]
+                    # Prefer Indices over Equities for similar names if needed, 
+                    # but usually unique.
+                    if "-INDEX" in matched_spot: 
+                        break 
+            
+            if matched_spot:
+                root_spot = matched_spot
+            else:
+                # Fallback for Stocks (RELIANCE, HDFCBANK)
+                # If no CM description match (rare), construct standard Equity symbol
+                # e.g. NSE:RELIANCE24... -> NSE:RELIANCE-EQ
+                try:
+                    # Strip Exchange
+                    parts = sym.split(':')
+                    raw_sym = parts[1] if len(parts) > 1 else sym
                     
-                temp_db[root] = { "spot": spot_sym, "exch": exch, "items": [] }
+                    # Heuristic: Extract alpha prefix before numbers/dates
+                    # Simple regex-like logic manually
+                    import re
+                    # Match standard FO format: SYMBOL + DD + MON...
+                    # We just take the non-digit prefix for most stocks
+                    m = re.match(r"([A-Z\-\&]+)", raw_sym)
+                    if m:
+                        short_name = m.group(1)
+                        if exch == "NSE":
+                            root_spot = f"NSE:{short_name}-EQ"
+                        elif exch == "BSE":
+                            root_spot = f"BSE:{short_name}"
+                        elif exch == "MCX":
+                            # MCX often uses the Future as reference, leave blank to let client decide
+                            root_spot = "" 
+                except:
+                    root_spot = sym # Fallback
+            
+            # If we found a valid root spot (e.g. NSE:NIFTY50-INDEX), use that as the Key
+            # This ensures the Search returns the EXACT Instrument ID.
+            
+            # If MCX/Commodity, we often group by the root name (CRUDEOIL) 
+            # because Spot symbols aren't always traded or consistent.
+            if exch == "MCX":
+                # Use simple root name for grouping key, but empty spot implies logic in v16.txt handles it
+                match_mcx = desc_upper.split(' ')[0]
+                db_key = match_mcx # e.g. CRUDEOIL
+                # v16.txt logic: "If Spot is missing, default to Future". 
+                # So we leave spot empty for MCX unless we really know it.
+                if not root_spot: root_spot = "" 
+            else:
+                # For NSE/BSE, the Key IS the Spot Symbol (The "Exact Value")
+                if root_spot:
+                    db_key = root_spot
+                else:
+                    # Fallback to description first word if all else fails
+                    db_key = desc_parts[0]
+
+            if db_key not in temp_db:
+                temp_db[db_key] = { 
+                    "spot": root_spot, 
+                    "exch": exch, 
+                    "items": [] 
+                }
 
             try:
                 exp = int(row['Expiry'])
@@ -145,24 +250,29 @@ def update_master_db():
                     except:
                         pass
             
-            temp_db[root]["items"].append({
+            temp_db[db_key]["items"].append({
                 "s": sym, "e": exp, "k": strike, "t": opt_type
             })
 
         final_db = {}
         search_list = []
         
-        for root, data in temp_db.items():
+        for key, data in temp_db.items():
+            # Filter out junk groups with no items
+            if not data["items"]: continue
+            
             sorted_items = sorted(data["items"], key=lambda x: (x['e'], x['k']))
-            final_db[root] = { 
+            final_db[key] = { 
                 "spot": data["spot"], 
                 "exch": data["exch"],
                 "items": sorted_items 
             }
+            
+            # Search Result: Display the Key exactly as it is (e.g. NSE:NIFTY50-INDEX)
             search_list.append({
-                "root": root,
+                "root": key,            # This is sent to v16.txt
                 "exch": data["exch"],
-                "display": f"{root}"
+                "display": key          # User sees this
             })
 
         MASTER_DB = final_db
@@ -259,17 +369,21 @@ def search_symbol(q: str = Query(..., min_length=1)):
         return JSONResponse([])
     
     query = q.upper()
-    results = [x for x in SEARCH_INDEX if x['root'].startswith(query)]
-    if len(results) < 10:
-        results += [x for x in SEARCH_INDEX if query in x['root'] and x not in results]
-        
+    # Filter: Check if query is in Root or Display
+    results = [x for x in SEARCH_INDEX if query in x['root'] or query in x['display']]
+    
+    # Priority sorting: Starts with query first
+    results.sort(key=lambda x: 0 if x['root'].startswith(query) else 1)
+    
     return JSONResponse(results[:20])
 
 @app.get("/api/chain")
 def get_chain(symbol: str):
-    root = symbol.upper()
+    root = symbol.upper() # This will now receive "NSE:NIFTY50-INDEX" directly
     if root in MASTER_DB:
         return JSONResponse(MASTER_DB[root])
+    # Fallback: Try identifying simple root if full symbol fails (compatibility)
+    # But usually, v16 sends exactly what search returns
     return JSONResponse({"error": "Symbol not found"}, status_code=404)
 
 @app.get("/api/refresh")
@@ -725,7 +839,7 @@ HTML_TEMPLATE = """
             const status = await res.json();
             if(status.ready) {
                 document.getElementById('loader').style.display = 'none';
-                if(!CURRENT_ROOT) loadSymbol('NIFTY'); 
+                if(!CURRENT_ROOT) loadSymbol('NIFTY 50'); 
             } else {
                 document.getElementById('loaderMsg').innerText = status.message.toUpperCase();
                 setTimeout(checkStatus, 2000);
@@ -764,7 +878,9 @@ HTML_TEMPLATE = """
         document.getElementById('loader').style.display = 'flex';
         
         try {
-            const res = await fetch(`/api/chain?symbol=${root}`);
+            // Encode root because it may contain characters like ':'
+            const encRoot = encodeURIComponent(root);
+            const res = await fetch(`/api/chain?symbol=${encRoot}`);
             if(!res.ok) throw new Error("Symbol not found");
             const data = await res.json();
             
@@ -775,9 +891,13 @@ HTML_TEMPLATE = """
 
             FUT_SYMS = CHAIN_DATA.filter(x => x.t === 'FUT').sort((a,b) => a.e - b.e).slice(0, 3);
             
-            if(SPOT_SYM === "MCX" && FUT_SYMS.length > 0) SPOT_SYM = FUT_SYMS[0].s; 
+            if(!SPOT_SYM && FUT_SYMS.length > 0) {
+                 // Fallback if spot is empty (e.g. MCX), use Future
+                 SPOT_SYM = FUT_SYMS[0].s; 
+            }
             
-            document.getElementById('spotLabel').innerText = (SPOT_SYM.includes('INDEX') || SPOT_SYM.includes('-EQ') || SPOT_SYM.startsWith('BSE:')) ? "SPOT PRICE" : "FUT REFERENCE";
+            const isIndex = SPOT_SYM.includes('INDEX') || SPOT_SYM.includes('-EQ') || SPOT_SYM.startsWith('BSE:');
+            document.getElementById('spotLabel').innerText = isIndex ? "SPOT PRICE" : "FUT REFERENCE";
 
             setupExpiries();
             document.getElementById('loader').style.display = 'none';
@@ -863,7 +983,11 @@ HTML_TEMPLATE = """
         IS_POLLING = true;
         
         try {
-            let symbolsToFetch = [SPOT_SYM, ...FUT_SYMS.map(x => x.s)];
+            // Ensure SPOT_SYM is valid (not empty) before requesting
+            let symbolsToFetch = [];
+            if(SPOT_SYM && SPOT_SYM.length > 2) symbolsToFetch.push(SPOT_SYM);
+            FUT_SYMS.forEach(f => { if(f.s) symbolsToFetch.push(f.s) });
+
             const kList = Array.from(document.querySelectorAll('.row')).map(r => parseFloat(r.dataset.k));
             
             if(kList.length > 0) {