Update app.py

app.py

@@ -7,7 +7,7 @@ import time
 import urllib.parse
 import os
 import threading
-from datetime import datetime
+from datetime import datetime, timezone
 from fastapi import FastAPI, Request, Response, Cookie, BackgroundTasks, Query
 from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
 from typing import Optional, List, Dict
@@ -17,12 +17,11 @@ from typing import Optional, List, Dict
 # ==========================================
 CLIENT_ID    = "J4PNT5QI22-100" 
 SECRET_KEY   = "ABRSWC1PMR"
-# IMPORTANT: Ensure this matches your Space URL exactly
 REDIRECT_URI = "https://topsecrettraders-depthchainindia.hf.space/callback" 
 
 CACHE_FILE = "master_cache.json"
 
-# --- SUPABASE CONFIG (FROM v16.txt) ---
+# --- SUPABASE CONFIG ---
 SUPABASE_URL = "https://tuyhtqxfvjndbyjazosi.supabase.co"
 SUPABASE_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InR1eWh0cXhmdmpuZGJ5amF6b3NpIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc2ODE0NjAyMiwiZXhwIjoyMDgzNzIyMDIyfQ.YsmGS67vL5TghB5OOi6pmsPL6rrgx7SNpARHmQ9b5cw"
 
@@ -212,18 +211,29 @@ def startup_event():
 # 3. ROUTES & HISTORICAL LOGIC
 # ==========================================
 
+def date_str_to_epoch(date_str: str) -> int:
+    """Converts YYYY-MM-DD string to UNIX Timestamp (Seconds)"""
+    if not date_str: return 0
+    try:
+        dt = datetime.strptime(date_str, "%Y-%m-%d")
+        return int(dt.replace(tzinfo=timezone.utc).timestamp())
+    except:
+        return 0
+
 def decompress_payload(rich_data: List[Dict]) -> Dict:
     """
     Converts the v16.txt rich array format back to Fyers Data format.
-    Input: [ { "u": "NSE:...", "p": 100, "v": 50, "q": [10, 20], "o": [5, 0] }, ... ]
-    Output: { "NSE:...": { "ltp": 100, "v": 50, "totalbuyqty": 10, ... } }
+    Input: [ { "u": "NSE:...", "p": 100, "v": 50, "q": [10, 20], "o": [5, 0], "e": "2025-02-26" }, ... ]
+    Output: { "NSE:...": { "ltp": 100, "v": 50, "totalbuyqty": 10, "expiry": 1740528000 ... } }
     """
     fyers_data = {}
     for item in rich_data:
         sym = item.get('u')
         if not sym: continue
         
-        # Mapping based on v16.txt logic
+        expiry_str = item.get('e', None)
+        expiry_ts = date_str_to_epoch(expiry_str) if expiry_str else 0
+
         fyers_data[sym] = {
             "ltp": item.get('p', 0),
             "v": item.get('v', 0),
@@ -231,8 +241,7 @@ def decompress_payload(rich_data: List[Dict]) -> Dict:
             "totalsellqty": item.get('q', [0,0])[1],
             "oi": item.get('o', [0,0])[0],
             "oipercent": item.get('o', [0,0])[1],
-            # Helper for expiry if stored in payload (v16 handles this)
-            "expiry": item.get('e', None) 
+            "expiry": expiry_ts # Converted to Epoch for UI compatibility
         }
     return fyers_data
 
@@ -311,55 +320,78 @@ def force_refresh(background_tasks: BackgroundTasks):
     background_tasks.add_task(update_master_db)
     return JSONResponse({"s": "ok"})
 
-# --- HISTORICAL API ENDPOINTS ---
+# --- HISTORICAL API ENDPOINTS WITH FULL PAGINATION ---
 
 @app.get("/api/hist/dates")
 def get_hist_dates(symbol: str):
-    """Fetches unique dates available for a symbol from Supabase"""
+    """Fetches ALL available dates for a symbol using pagination"""
     try:
-        # We fetch recorded_at, convert to date, distinct
-        # Supabase doesn't support easy 'distinct' on derived columns easily in python client without rpc
-        # So we fetch 'time_signature' which is string "YYYY-MM-DD HH:MM:SS"
-        # We'll just fetch entries, order by recency, and extract dates.
-        # Limit to last 500 records to extract unique dates
-        res = supabase.table('snapshots')\
-            .select('time_signature')\
-            .eq('symbol', symbol.upper())\
-            .order('recorded_at', desc=True)\
-            .limit(1000)\
-            .execute()
-        
         dates = set()
-        for item in res.data:
-            ts_str = item.get('time_signature', '')
-            if ts_str:
-                dates.add(ts_str.split(' ')[0]) # Extract YYYY-MM-DD
-        
+        offset = 0
+        limit = 1000 # Supabase max
+        
+        while True:
+            # Loop to fetch ALL records
+            res = supabase.table('snapshots')\
+                .select('time_signature')\
+                .eq('symbol', symbol.upper())\
+                .order('recorded_at', desc=True)\
+                .range(offset, offset + limit - 1)\
+                .execute()
+            
+            data = res.data
+            if not data:
+                break
+                
+            for item in data:
+                ts_str = item.get('time_signature', '')
+                if ts_str:
+                    dates.add(ts_str.split(' ')[0]) # Extract YYYY-MM-DD
+            
+            if len(data) < limit:
+                break
+            
+            offset += limit
+            # Safety break at 50k rows to prevent timeout if excessive
+            if offset > 100000: break 
+
         return JSONResponse(sorted(list(dates), reverse=True))
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
 
 @app.get("/api/hist/minutes")
 def get_hist_minutes(symbol: str, date: str):
-    """Fetches available minutes for a specific date"""
+    """Fetches ALL available minutes for a specific date using pagination"""
     try:
-        # Filter strings starting with date
-        res = supabase.table('snapshots')\
-            .select('time_signature')\
-            .eq('symbol', symbol.upper())\
-            .like('time_signature', f'{date}%')\
-            .order('recorded_at', desc=True)\
-            .execute()
-            
         minutes = set()
-        for item in res.data:
-            ts_str = item.get('time_signature', '')
-            # Format: YYYY-MM-DD HH:MM:SS -> Extract HH:MM
-            parts = ts_str.split(' ')
-            if len(parts) > 1:
-                time_part = parts[1] # HH:MM:SS
-                minutes.add(time_part[:5]) # HH:MM
+        offset = 0
+        limit = 1000
+        
+        while True:
+            res = supabase.table('snapshots')\
+                .select('time_signature')\
+                .eq('symbol', symbol.upper())\
+                .like('time_signature', f'{date}%')\
+                .order('recorded_at', desc=True)\
+                .range(offset, offset + limit - 1)\
+                .execute()
                 
+            data = res.data
+            if not data:
+                break
+                
+            for item in data:
+                ts_str = item.get('time_signature', '')
+                # Format: YYYY-MM-DD HH:MM:SS -> Extract HH:MM
+                parts = ts_str.split(' ')
+                if len(parts) > 1:
+                    time_part = parts[1] # HH:MM:SS
+                    minutes.add(time_part[:5]) # HH:MM
+            
+            if len(data) < limit:
+                break
+            offset += limit
+
         return JSONResponse(sorted(list(minutes), reverse=True))
     except Exception as e:
         return JSONResponse({"error": str(e)}, status_code=500)
@@ -371,32 +403,45 @@ def get_hist_data(symbol: str, date: str, minute: str):
     Returns a list of { time: "SS", data: {...reconstructed...} }
     """
     try:
-        # Construct query pattern like "2025-01-01 09:15%"
         query_sig = f"{date} {minute}"
-        
-        res = supabase.table('snapshots')\
-            .select('time_signature, payload')\
-            .eq('symbol', symbol.upper())\
-            .like('time_signature', f'{query_sig}%')\
-            .order('recorded_at', desc=False)\
-            .execute()
-            
         timeline = []
-        for item in res.data:
-            ts = item.get('time_signature', '')
-            sec = ts.split(':')[-1] if ':' in ts else "00"
+        offset = 0
+        limit = 1000
+        
+        while True:
+            res = supabase.table('snapshots')\
+                .select('time_signature, payload')\
+                .eq('symbol', symbol.upper())\
+                .like('time_signature', f'{query_sig}%')\
+                .order('recorded_at', desc=False)\
+                .range(offset, offset + limit - 1)\
+                .execute()
             
-            raw_payload = item.get('payload')
-            # Handle if payload is stringified JSON
-            if isinstance(raw_payload, str):
-                raw_payload = json.loads(raw_payload)
+            data = res.data
+            if not data:
+                break
                 
-            clean_data = decompress_payload(raw_payload)
-            timeline.append({
-                "s": sec,
-                "d": clean_data
-            })
+            for item in data:
+                ts = item.get('time_signature', '')
+                sec = ts.split(':')[-1] if ':' in ts else "00"
+                
+                raw_payload = item.get('payload')
+                if isinstance(raw_payload, str):
+                    try:
+                        raw_payload = json.loads(raw_payload)
+                    except:
+                        raw_payload = []
+                        
+                clean_data = decompress_payload(raw_payload)
+                timeline.append({
+                    "s": sec,
+                    "d": clean_data
+                })
             
+            if len(data) < limit:
+                break
+            offset += limit
+
         return JSONResponse({"s": "ok", "timeline": timeline})
         
     except Exception as e: