Create app.py v1

app.py v1

@@ -0,0 +1,754 @@
+import pandas as pd
+import numpy as np
+import requests
+import uvicorn
+import asyncio
+import re
+import uuid
+import json
+import io
+from datetime import timedelta
+from fastapi import FastAPI, Body, HTTPException
+from fastapi.responses import HTMLResponse, JSONResponse
+from huggingface_hub import HfApi, hf_hub_download
+from typing import Optional, List
+
+# ==========================================
+# 1. CONFIGURATION & SECRETS
+# ==========================================
+SB_URL = "https://ivyyfhymhdheykpntozd.supabase.co"
+SB_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Iml2eXlmaHltaGRoZXlrcG50b3pkIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImlhdCI6MTc4Mzg5NzAwOCwiZXhwIjoyMDk5NDczMDA4fQ.cHvePFN3NML8Zv12D2vV91MigJGRerZvD_Cevbu1AFk"
+HEADERS = {
+    "apikey": SB_KEY,
+    "Authorization": f"Bearer {SB_KEY}",
+    "Content-Type": "application/json",
+    "Prefer": "return=representation"
+}
+
+HF_TOKEN_PART_1 = "hf_EzaiwOHiejWFLN"
+HF_TOKEN_PART_2 = "otvQqTPUTksQQfYxrbJg"
+HF_TOKEN = HF_TOKEN_PART_1 + HF_TOKEN_PART_2
+HF_REPO_ID = "topsecrettraders/DepthChain"
+
+hf_api = HfApi(token=HF_TOKEN)
+app = FastAPI()
+
+# IN-MEMORY CACHE (Replaces local disk storage to prevent HF Space File Deletion issues)
+SESSION_CACHE = {}
+
+# ==========================================
+# 2. SUPABASE HELPER FUNCTIONS
+# ==========================================
+def sb_rpc(func_name: str, params: dict = None):
+    if params is None: params = {}
+    try:
+        url = f"{SB_URL}/rest/v1/rpc/{func_name}"
+        r = requests.post(url, headers=HEADERS, json=params)
+        r.raise_for_status()
+        return r.json()
+    except Exception as e:
+        print(f"DB Error RPC ({func_name}): {e}")
+        return[]
+
+# ==========================================
+# 3. API ROUTES - METADATA & UI
+# ==========================================
+@app.get("/")
+def home():
+    return HTMLResponse(HTML_TEMPLATE)
+
+@app.get("/api/dates")
+def get_dates():
+    data = sb_rpc("get_available_dates", {"p_limit": 50})
+    return JSONResponse([row['available_date'] for row in data])
+
+@app.get("/api/roots")
+def get_roots(date: str):
+    data = sb_rpc("get_roots_for_date", {"p_date": date})
+    return JSONResponse([row['root_name'] for row in data])
+
+@app.get("/api/expiries")
+def get_expiries(date: str, root: str):
+    data = sb_rpc("get_expiries_for_root", {"p_date": date, "p_root": root.upper()})
+    return JSONResponse([row['expiry_date'] for row in data])
+
+# ==========================================
+# 4. HUGGING FACE EXPLORER API
+# ==========================================
+@app.get("/api/hf_files")
+def list_hf_files():
+    try:
+        files = hf_api.list_repo_files(repo_id=HF_REPO_ID, repo_type="dataset")
+        folders = set()
+        for f in files:
+            if f.startswith("Market Data/") and f.endswith("data.parquet"):
+                folders.add(f.replace("/data.parquet", ""))
+        return JSONResponse({"status": "success", "folders": sorted(list(folders), reverse=True)})
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": str(e)})
+
+@app.get("/api/hf_metadata")
+def get_hf_metadata(folder_path: str):
+    try:
+        meta_path = f"{folder_path}/metadata.json"
+        local_path = hf_hub_download(repo_id=HF_REPO_ID, filename=meta_path, repo_type="dataset", token=HF_TOKEN)
+        with open(local_path, "r") as f:
+            metadata = json.load(f)
+        return JSONResponse({"status": "success", "metadata": metadata})
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": "Metadata not found or error fetching."})
+
+@app.get("/api/view_parquet")
+def view_parquet(folder_path: str):
+    try:
+        file_path = f"{folder_path}/data.parquet"
+        local_path = hf_hub_download(repo_id=HF_REPO_ID, filename=file_path, repo_type="dataset", token=HF_TOKEN)
+        df = pd.read_parquet(local_path)
+        return JSONResponse({"status": "success", "columns": df.columns.tolist(), "data": df.to_dict(orient="records")})
+    except Exception as e:
+        return JSONResponse({"status": "error", "message": str(e)})
+
+# ==========================================
+# 5. STEP 1: FETCH & INSPECT
+# ==========================================
+@app.post("/api/inspect_data")
+async def inspect_data(
+    date: str = Body(...),
+    root: str = Body(...),
+    expiry: str = Body(...),
+    start_time: str = Body("09:15"),
+    end_time: str = Body("15:30"),
+    atm_range: int = Body(10),
+    required_vars: List[str] = Body(["P", "CB", "PS", "CS", "PB"])
+):
+    root = root.upper()
+    # STRICT BOUNDS: Exactly the minute specified, no bleeding into trailing seconds.
+    t_start_dt = pd.to_datetime(f"{date} {start_time}:00")
+    t_end_dt = pd.to_datetime(f"{date} {end_time}:00")
+
+    inst_data = sb_rpc("get_instruments_for_expiry", {"p_date": date, "p_root": root, "p_expiry": expiry})
+    instruments = [r['instrument_name'] for r in inst_data]
+
+    if not instruments:
+        return JSONResponse({"error": f"No data found in Supabase for {root} {expiry} on {date}"})
+
+    spot_keys =[k for k in instruments if k.endswith('-INDEX') or k.endswith('-EQ')]
+    fut_keys =[k for k in instruments if 'FUT' in k]
+    ref_instrument = spot_keys[0] if spot_keys else (fut_keys[0] if fut_keys else instruments[0])
+
+    strike_map = {}
+    for inst in instruments:
+        if inst.endswith('CE') or inst.endswith('PE'):
+            clean_inst = inst.split(":")[-1]
+            if clean_inst.startswith(root):
+                match = re.search(r'(\d+(?:\.\d+)?)(?:CE|PE)$', clean_inst[len(root)+5:])
+                if match:
+                    strike_map[inst] = float(match.group(1))
+                    continue
+            matches = re.findall(r'(\d+(?:\.\d+)?)(?=CE$|PE$)', clean_inst)
+            if matches: strike_map[inst] = float(matches[-1])
+
+    base_params = {
+        "p_root": root, "p_expiry": expiry, "p_ref_instrument": ref_instrument,
+        "p_atm_range": atm_range, "p_strike_map": strike_map
+    }
+
+    async def fetch_chunk(start_dt, end_dt, retries=3):
+        params = base_params.copy()
+        params["p_start_time"] = start_dt.strftime("%Y-%m-%d %H:%M:%S")
+        params["p_end_time"] = end_dt.strftime("%Y-%m-%d %H:%M:%S")
+        loop = asyncio.get_event_loop()
+        for attempt in range(retries):
+            try:
+                return await loop.run_in_executor(None, sb_rpc, "export_advanced_10s_data", params)
+            except Exception as e:
+                if attempt == retries - 1: return[]
+                await asyncio.sleep(1)
+
+    tasks =[]
+    curr = t_start_dt
+    while curr <= t_end_dt:
+        nxt = curr + timedelta(hours=1) - timedelta(seconds=1)
+        if nxt > t_end_dt: nxt = t_end_dt
+        tasks.append(fetch_chunk(curr, nxt))
+        curr += timedelta(hours=1)
+
+    chunk_results = await asyncio.gather(*tasks)
+    
+    all_records =[]
+    for res in chunk_results:
+        if isinstance(res, list): all_records.extend(res)
+
+    if not all_records:
+        return JSONResponse({"error": "No valid records fetched. Data might be empty for this date/time."})
+
+    df = pd.DataFrame(all_records)
+    df.rename(columns={"tick_ts": "timestamp", "price": "P", "cb": "CB", "cs": "CS", "pb": "PB", "ps": "PS", "b_qty": "B", "s_qty": "S", "vol": "V", "oi": "OI"}, inplace=True)
+    df['timestamp'] = pd.to_datetime(df['timestamp'])
+    df.drop_duplicates(subset=['timestamp'], inplace=True)
+    df.set_index('timestamp', inplace=True)
+    df.sort_index(inplace=True)
+
+    # 1. STRICT FILTER: Remove anything outside user bounds that the chunking may have dragged in
+    df = df.loc[(df.index >= t_start_dt) & (df.index <= t_end_dt)]
+    if df.empty:
+        return JSONResponse({"error": "Data exists, but completely outside your specified Start/End Time."})
+
+    available_vars =[v for v in required_vars if v in df.columns]
+    df = df[available_vars].copy()
+
+    # 2. DYNAMIC INDEX: Only pad between the actual bounds of fetched data.
+    # No more faking empty timestamps before the broker actually started sending data.
+    actual_min = df.index.min()
+    actual_max = df.index.max()
+    
+    full_idx = pd.date_range(start=actual_min, end=actual_max, freq='10s')
+    
+    raw_ticks = len(df)
+    expected_ticks = len(full_idx)
+    missing_ts = full_idx.difference(df.index).to_pydatetime()
+    
+    missing_blocks =[]
+    if len(missing_ts) > 0:
+        start_block = missing_ts[0]
+        prev = missing_ts[0]
+        for ts in missing_ts[1:]:
+            if (ts - prev).total_seconds() > 10.5:
+                missing_blocks.append({
+                    "start": start_block.strftime('%Y-%m-%d %H:%M:%S'),
+                    "end": prev.strftime('%Y-%m-%d %H:%M:%S'),
+                    "count": int((prev - start_block).total_seconds()/10) + 1
+                })
+                start_block = ts
+            prev = ts
+        missing_blocks.append({
+            "start": start_block.strftime('%Y-%m-%d %H:%M:%S'),
+            "end": prev.strftime('%Y-%m-%d %H:%M:%S'),
+            "count": int((prev - start_block).total_seconds()/10) + 1
+        })
+
+    # For the UI, we reindex to correctly identify all upcoming NaNs
+    df_reindexed = df.reindex(full_idx)
+    
+    null_details_existing = {}
+    for col in available_vars:
+        # We flag 0.0 as an error because Flow/Volume/Price are cumulative and should never drop to 0 mid-day.
+        na_times = df_reindexed[df_reindexed[col].isna() | (df_reindexed[col] == 0)].index.strftime('%H:%M:%S').tolist()
+        null_details_existing[col] = {"count": len(na_times), "times": na_times[:50]} 
+
+    # Cache original df (not the padded one, so trim operations work natively)
+    session_id = str(uuid.uuid4())
+    SESSION_CACHE[session_id] = df.copy()
+
+    return JSONResponse({
+        "status": "success",
+        "session_id": session_id,
+        "stats": {
+            "expected_ticks": expected_ticks,
+            "found_ticks": raw_ticks,
+            "missing_time_ticks": len(missing_ts),
+            "missing_blocks": missing_blocks,
+            "null_breakdown": null_details_existing,
+            "variables": available_vars,
+            "original_start": actual_min.strftime('%Y-%m-%d %H:%M:%S'),
+            "original_end": actual_max.strftime('%Y-%m-%d %H:%M:%S')
+        },
+        "meta": {
+            "date": date, "root": root, "expiry": expiry,
+            "start": start_time, "end": end_time, "atm_range": atm_range
+        }
+    })
+
+# ==========================================
+# 6. STEP 2: CLEAN & ARCHIVE
+# ==========================================
+@app.post("/api/clean_and_upload")
+async def clean_and_upload(
+    session_id: str = Body(...),
+    fill_method: str = Body("ffill"),
+    trim_start: str = Body(None),
+    trim_end: str = Body(None),
+    metadata: dict = Body(...)
+):
+    if session_id not in SESSION_CACHE:
+        return JSONResponse({"error": "Session expired or invalid. Please Fetch & Inspect again."})
+
+    df = SESSION_CACHE[session_id].copy()
+
+    # 1. Apply Trimming boundaries
+    t_start = pd.to_datetime(trim_start) if trim_start else df.index.min()
+    t_end = pd.to_datetime(trim_end) if trim_end else df.index.max()
+    
+    df = df.loc[(df.index >= t_start) & (df.index <= t_end)]
+
+    # 2. Enforce Strict 10-Second Indexing on the trimmed range
+    full_idx = pd.date_range(start=t_start, end=t_end, freq='10s')
+    df = df.reindex(full_idx)
+
+    # 3. Replace 0.0 with NaN! (Crucial so FFill actually fixes the glitches)
+    df.replace(0, np.nan, inplace=True)
+
+    # 4. Apply Imputation Strategy
+    if fill_method == "ffill":
+        df_clean = df.ffill().fillna(0)  
+    elif fill_method == "interpolate":
+        df_clean = df.interpolate(method='linear').fillna(0)
+    elif fill_method == "zero":
+        df_clean = df.fillna(0)
+    else:
+        df_clean = df.ffill().fillna(0)
+
+    df_clean.reset_index(inplace=True)
+    df_clean.rename(columns={"index": "timestamp"}, inplace=True)
+    df_clean['timestamp'] = df_clean['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
+
+    metadata["imputation_applied"] = fill_method
+    metadata["trim_start_applied"] = trim_start
+    metadata["trim_end_applied"] = trim_end
+    metadata["total_final_rows"] = len(df_clean)
+
+    date = metadata["date"]
+    root = metadata["root"]
+    expiry = metadata["expiry"]
+    hf_folder = f"Market Data/{date}/{root}/{expiry}"
+
+    try:
+        parquet_buffer = io.BytesIO()
+        df_clean.to_parquet(parquet_buffer, index=False)
+        parquet_buffer.seek(0)
+        
+        json_buffer = io.BytesIO(json.dumps(metadata, indent=4).encode('utf-8'))
+        json_buffer.seek(0)
+
+        hf_api.upload_file(path_or_fileobj=parquet_buffer, path_in_repo=f"{hf_folder}/data.parquet", repo_id=HF_REPO_ID, repo_type="dataset")
+        hf_api.upload_file(path_or_fileobj=json_buffer, path_in_repo=f"{hf_folder}/metadata.json", repo_id=HF_REPO_ID, repo_type="dataset")
+        
+        del SESSION_CACHE[session_id]
+
+        return JSONResponse({"status": "success", "message": "Cleaned data & Metadata archived to Hugging Face!"})
+    except Exception as e:
+        return JSONResponse({"error": f"Upload Failed: {str(e)}"})
+
+
+# ==========================================
+# 7. FRONTEND TEMPLATE
+# ==========================================
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>DepthChain Data Engineering</title>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
+    <style>
+        :root { --bg: #0d1117; --panel: #161b22; --border: #30363d; --text: #e6edf3; --accent: #2f81f7; --success: #238636; --danger: #f85149; --warn: #d29922; }
+        body { background: var(--bg); color: var(--text); font-family: 'Inter', sans-serif; margin: 0; padding: 20px; font-size: 13px; }
+        .grid { display: grid; grid-template-columns: 1fr 1.2fr; gap: 20px; max-width: 1300px; margin: 0 auto; }
+        .card { background: var(--panel); border: 1px solid var(--border); border-radius: 8px; padding: 20px; display: flex; flex-direction: column; }
+        h2 { margin-top: 0; font-size: 15px; border-bottom: 1px solid var(--border); padding-bottom: 10px; color: var(--accent); }
+        label { display: block; margin-bottom: 6px; color: #8b949e; font-size: 11px; font-weight: 600; text-transform: uppercase; }
+        select, input { width: 100%; padding: 8px 12px; margin-bottom: 12px; background: var(--bg); border: 1px solid var(--border); color: var(--text); border-radius: 6px; font-size: 13px; box-sizing: border-box; }
+        button { background: var(--accent); color: white; border: none; padding: 10px; border-radius: 6px; font-weight: bold; cursor: pointer; transition: 0.2s; width: 100%; }
+        button:hover { background: #58a6ff; }
+        button:disabled { background: var(--border); color: #8b949e; cursor: not-allowed; }
+        .btn-success { background: var(--success); }
+        .btn-success:hover { background: #2ea043; }
+        .btn-warn { background: transparent; border: 1px solid var(--warn); color: var(--warn); padding: 4px 8px; font-size: 11px; border-radius: 4px; width: auto; }
+        .btn-warn:hover { background: rgba(210, 153, 34, 0.2); }
+        
+        .row { display: flex; gap: 10px; }
+        .col { flex: 1; }
+        
+        .stat-box { background: var(--bg); border: 1px solid var(--border); border-radius: 6px; padding: 15px; margin-bottom: 15px; }
+        .stat-item { display: flex; justify-content: space-between; margin-bottom: 8px; border-bottom: 1px dashed var(--border); padding-bottom: 4px; align-items: center; }
+        .stat-val { font-family: monospace; font-size: 13px; font-weight: bold; }
+        .val-good { color: var(--success); }
+        .val-bad { color: var(--danger); }
+        .val-warn { color: var(--warn); }
+        
+        .hf-list { flex: 1; overflow-y: auto; background: var(--bg); border: 1px solid var(--border); border-radius: 6px; padding: 10px; max-height: 400px; }
+        .hf-item { padding: 8px; border-bottom: 1px solid var(--border); display: flex; justify-content: space-between; align-items: center; }
+        .hf-item:hover { background: #21262d; }
+        .hf-btns { display: flex; gap: 6px; }
+        .hf-btn { cursor: pointer; font-size: 11px; padding: 4px 8px; background: var(--panel); border: 1px solid var(--border); color: var(--text); border-radius: 4px; }
+        .hf-btn:hover { border-color: var(--accent); color: var(--accent); }
+        
+        .toast { position: fixed; bottom: 20px; right: 20px; background: var(--success); color: white; padding: 12px 20px; border-radius: 6px; display: none; box-shadow: 0 4px 12px rgba(0,0,0,0.5); z-index: 9999; }
+        
+        details { margin-bottom: 5px; font-size: 12px; }
+        summary { cursor: pointer; color: var(--accent); font-weight: bold; margin-bottom: 5px; outline: none; }
+        .inspect-times { font-family: monospace; font-size: 10px; color: #8b949e; background: #000; padding: 6px; border-radius: 4px; max-height: 100px; overflow-y: auto; }
+
+        .modal-bg { position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.8); display: none; justify-content: center; align-items: center; z-index: 1000; }
+        .modal-content { background: var(--panel); border: 1px solid var(--border); border-radius: 8px; width: 90%; max-width: 1000px; height: 85vh; display: flex; flex-direction: column; }
+        .modal-header { padding: 15px; border-bottom: 1px solid var(--border); display: flex; justify-content: space-between; align-items: center; font-weight: bold; }
+        .modal-body { flex: 1; padding: 15px; overflow: auto; }
+        .close-btn { background: transparent; border: none; color: var(--danger); font-size: 18px; cursor: pointer; width: auto; padding: 0;}
+        
+        table.data-table { width: 100%; border-collapse: collapse; font-family: monospace; font-size: 11px; text-align: right; }
+        table.data-table th { background: var(--bg); color: #8b949e; padding: 8px; border-bottom: 1px solid var(--border); position: sticky; top: 0; }
+        table.data-table td { padding: 6px 8px; border-bottom: 1px solid #21262d; }
+        table.data-table tr:hover { background: #21262d; }
+    </style>
+</head>
+<body>
+
+<div class="grid">
+    <!-- LEFT COLUMN -->
+    <div style="display: flex; flex-direction: column; gap: 20px;">
+        <div class="card">
+            <h2>1. Extract & Inspect Data</h2>
+            <div class="row">
+                <div class="col"><label>Date</label><select id="selDate" onchange="App.loadRoots()"></select></div>
+                <div class="col"><label>Symbol</label><select id="selRoot" onchange="App.loadExpiries()"></select></div>
+            </div>
+            <label>Expiry Date</label><select id="selExpiry"></select>
+            
+            <div class="row">
+                <div class="col"><label>ATM Range (± N)</label><input type="number" id="inpAtm" value="10"></div>
+                <div class="col"><label>Variables</label><input type="text" id="inpVars" value="P,CB,PS,CS,PB"></div>
+            </div>
+            <div class="row">
+                <div class="col"><label>Strict Start</label><input type="time" id="inpStart" value="09:15"></div>
+                <div class="col"><label>Strict End</label><input type="time" id="inpEnd" value="15:30"></div>
+            </div>
+            
+            <button id="btnInspect" onclick="App.inspectData()">🔍 Fetch & Analyze Quality</button>
+        </div>
+
+        <div class="card" id="cardArchive" style="opacity: 0.4; pointer-events: none;">
+            <h2>2. Clean & Archive to Hugging Face</h2>
+            <div class="stat-box" id="statsBox">
+                <div style="text-align:center; color:#8b949e; font-style:italic;">Run Inspection first...</div>
+            </div>
+            
+            <label>Missing Data Handling Strategy (Imputation)</label>
+            <select id="selFill">
+                <option value="ffill">Forward Fill (Fixes Nulls & Zeroes with Previous Values) - REC.</option>
+                <option value="interpolate">Linear Interpolation (Average between gaps)</option>
+                <option value="zero">Fill with Zeroes</option>
+            </select>
+            
+            <button id="btnUpload" class="btn-success" onclick="App.uploadData()">☁️ Archive to Repository</button>
+        </div>
+    </div>
+
+    <!-- RIGHT COLUMN -->
+    <div class="card">
+        <div style="display:flex; justify-content:space-between; align-items:center;">
+            <h2>Hugging Face Repository Explorer</h2>
+            <button onclick="App.loadHFFolders()" style="width:auto; padding: 4px 12px; font-size:11px;">Refresh</button>
+        </div>
+        
+        <div class="hf-list" id="hfList">Loading...</div>
+        
+        <div class="stat-box" id="metaBox" style="margin-top: 15px; display: none;">
+            <h3 style="margin:0 0 10px 0; font-size:13px; color:var(--text);">Metadata Viewer</h3>
+            <pre id="metaContent" style="font-family:monospace; font-size:11px; color:#8b949e; margin:0; white-space:pre-wrap;"></pre>
+        </div>
+    </div>
+</div>
+
+<div class="toast" id="toast">✅ Success!</div>
+
+<!-- DATA VIEWER MODAL -->
+<div class="modal-bg" id="dataModal">
+    <div class="modal-content">
+        <div class="modal-header">
+            <span id="modalTitle">Data Viewer</span>
+            <button class="close-btn" onclick="document.getElementById('dataModal').style.display='none'">✖</button>
+        </div>
+        <div class="modal-body" id="modalBody">
+            Loading data table...
+        </div>
+    </div>
+</div>
+
+<script>
+const App = {
+    currentSession: null,
+    currentMeta: null,
+    trimStart: null,
+    trimEnd: null,
+    originalStartStr: null,
+    originalEndStr: null,
+    missingTimeTicks: 0,
+
+    async init() {
+        await this.loadDates();
+        await this.loadHFFolders();
+    },
+
+    showToast(msg) {
+        const t = document.getElementById('toast');
+        t.innerText = msg;
+        t.style.display = 'block';
+        setTimeout(() => t.style.display = 'none', 3000);
+    },
+
+    async loadDates() {
+        const res = await fetch('/api/dates');
+        const dates = await res.json();
+        document.getElementById('selDate').innerHTML = dates.map(d => `<option value="${d}">${d}</option>`).join('');
+        if(dates.length) this.loadRoots();
+    },
+
+    async loadRoots() {
+        const date = document.getElementById('selDate').value;
+        const res = await fetch(`/api/roots?date=${date}`);
+        const roots = await res.json();
+        document.getElementById('selRoot').innerHTML = roots.map(r => `<option value="${r}">${r}</option>`).join('');
+        if(roots.length) this.loadExpiries();
+    },
+
+    async loadExpiries() {
+        const date = document.getElementById('selDate').value;
+        const root = document.getElementById('selRoot').value;
+        if(!root) return;
+        const res = await fetch(`/api/expiries?date=${date}&root=${root}`);
+        const exps = await res.json();
+        document.getElementById('selExpiry').innerHTML = exps.map(e => `<option value="${e}">${e}</option>`).join('');
+    },
+
+    async inspectData() {
+        const btn = document.getElementById('btnInspect');
+        btn.disabled = true;
+        btn.innerText = "⏳ Fetching and Analyzing Chunks...";
+        
+        const payload = {
+            date: document.getElementById('selDate').value,
+            root: document.getElementById('selRoot').value,
+            expiry: document.getElementById('selExpiry').value,
+            atm_range: parseInt(document.getElementById('inpAtm').value),
+            required_vars: document.getElementById('inpVars').value.split(',').map(s=>s.trim()),
+            start_time: document.getElementById('inpStart').value,
+            end_time: document.getElementById('inpEnd').value
+        };
+
+        try {
+            const res = await fetch('/api/inspect_data', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(payload)
+            });
+            const data = await res.json();
+
+            if(data.error) {
+                alert(data.error);
+            } else {
+                this.currentSession = data.session_id;
+                this.currentMeta = data.meta;
+                this.currentMeta.stats = data.stats;
+                this.trimStart = data.stats.original_start;
+                this.trimEnd = data.stats.original_end;
+                this.originalStartStr = data.stats.original_start;
+                this.originalEndStr = data.stats.original_end;
+                this.missingTimeTicks = data.stats.missing_time_ticks;
+                
+                this.renderStats(data.stats);
+                
+                const c2 = document.getElementById('cardArchive');
+                c2.style.opacity = '1';
+                c2.style.pointerEvents = 'auto';
+            }
+        } catch(e) {
+            alert(`Error: ${e.message}`);
+        }
+        btn.disabled = false;
+        btn.innerText = "🔍 Fetch & Analyze Quality";
+    },
+
+    trimBlock(start, end, count, type, btnId) {
+        if (type === 'start') {
+            let endObj = new Date(end);
+            endObj.setSeconds(endObj.getSeconds() + 10);
+            this.trimStart = endObj.toISOString().replace('T',' ').substring(0, 19);
+        } else if (type === 'end') {
+            let startObj = new Date(start);
+            startObj.setSeconds(startObj.getSeconds() - 10);
+            this.trimEnd = startObj.toISOString().replace('T',' ').substring(0, 19);
+        }
+        
+        this.missingTimeTicks -= count;
+        document.getElementById('missingCountTxt').innerText = this.missingTimeTicks;
+        document.getElementById('missingCountTxt').className = "stat-val " + (this.missingTimeTicks === 0 ? 'val-good' : 'val-warn');
+        
+        document.getElementById('bndStart').innerText = this.trimStart;
+        document.getElementById('bndEnd').innerText = this.trimEnd;
+        
+        const b = document.getElementById(btnId);
+        b.innerText = "Trimmed ✔";
+        b.disabled = true;
+        b.style.textDecoration = "line-through";
+        b.style.color = "var(--success)";
+        b.style.borderColor = "var(--success)";
+    },
+
+    renderStats(s) {
+        const mClass = this.missingTimeTicks === 0 ? 'val-good' : (this.missingTimeTicks < 50 ? 'val-warn' : 'val-bad');
+        
+        let nullHtml = '';
+        for(let key in s.null_breakdown) {
+            let v = s.null_breakdown[key];
+            let c = v.count === 0 ? 'val-good' : 'val-bad';
+            
+            let details = v.count > 0 
+                ? `<details><summary>Inspect ${v.count} Nulls/Zeroes</summary><div class="inspect-times">${v.times.join(', ')}${v.count>50?'...':''}</div></details>` 
+                : `<div style="font-size:11px; color:var(--success)">Perfect Data</div>`;
+                
+            nullHtml += `<div class="stat-item" style="flex-direction:column; align-items:flex-start; margin-bottom:10px;">
+                <div style="display:flex; justify-content:space-between; width:100%;">
+                    <span>Glitched Nulls/Zeroes in ${key}</span><span class="stat-val ${c}">${v.count}</span>
+                </div>
+                ${details}
+            </div>`;
+        }
+
+        let blocksHtml = '';
+        if (s.missing_blocks.length > 0) {
+            blocksHtml = `<div style="margin-top:10px; font-weight:bold; font-size:11px; color:#8b949e; text-transform:uppercase;">Missing Time Ranges</div>`;
+            s.missing_blocks.forEach((b, i) => {
+                let btn = '';
+                if (b.start === this.originalStartStr) {
+                    btn = `<button id="tb_${i}" class="btn-warn" onclick="App.trimBlock('${b.start}', '${b.end}', ${b.count}, 'start', 'tb_${i}')">✂ Trim Start</button>`;
+                } else if (b.end === this.originalEndStr) {
+                    btn = `<button id="tb_${i}" class="btn-warn" onclick="App.trimBlock('${b.start}', '${b.end}', ${b.count}, 'end', 'tb_${i}')">✂ Trim End</button>`;
+                }
+                
+                blocksHtml += `<div class="stat-item" style="font-size:11px;">
+                    <span style="color:var(--warn)">${b.start.split(' ')[1]} ➔ ${b.end.split(' ')[1]} (${b.count} ticks)</span>
+                    ${btn}
+                </div>`;
+            });
+        }
+
+        document.getElementById('statsBox').innerHTML = `
+            <div class="stat-item"><span>Data Start Bound</span><span class="stat-val" id="bndStart" style="font-size:11px; color:var(--accent);">${this.trimStart}</span></div>
+            <div class="stat-item"><span>Data End Bound</span><span class="stat-val" id="bndEnd" style="font-size:11px; color:var(--accent);">${this.trimEnd}</span></div>
+            <div class="stat-item"><span>Raw Fetched Ticks</span><span class="stat-val">${s.found_ticks}</span></div>
+            <div class="stat-item"><span>Missing Ticks (Internal)</span><span class="stat-val ${mClass}" id="missingCountTxt">${this.missingTimeTicks}</span></div>
+            ${blocksHtml}
+            <div style="margin-top:15px; font-weight:bold; font-size:11px; color:#8b949e; text-transform:uppercase;">Value Quality Breakdown</div>
+            ${nullHtml}
+        `;
+    },
+
+    async uploadData() {
+        const btn = document.getElementById('btnUpload');
+        btn.disabled = true;
+        btn.innerText = "☁️ Imputing & Uploading to HF...";
+
+        try {
+            const res = await fetch('/api/clean_and_upload', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    session_id: this.currentSession,
+                    fill_method: document.getElementById('selFill').value,
+                    trim_start: this.trimStart,
+                    trim_end: this.trimEnd,
+                    metadata: this.currentMeta
+                })
+            });
+            const data = await res.json();
+
+            if(data.error) {
+                alert(data.error);
+            } else {
+                this.showToast(data.message);
+                
+                document.getElementById('cardArchive').style.opacity = '0.4';
+                document.getElementById('cardArchive').style.pointerEvents = 'none';
+                document.getElementById('statsBox').innerHTML = '<div style="text-align:center; color:#8b949e; font-style:italic;">Run Inspection first...</div>';
+                
+                this.loadHFFolders();
+            }
+        } catch(e) {
+            alert(`Error: ${e.message}`);
+        }
+        btn.disabled = false;
+        btn.innerText = "☁️ Archive to Repository";
+    },
+
+    async loadHFFolders() {
+        document.getElementById('hfList').innerHTML = 'Fetching...';
+        const res = await fetch('/api/hf_files');
+        const data = await res.json();
+        
+        if(data.status === 'error') {
+            document.getElementById('hfList').innerHTML = `<span style="color:red">${data.message}</span>`;
+            return;
+        }
+
+        const folders = data.folders;
+        if(!folders.length) {
+            document.getElementById('hfList').innerHTML = '<div style="color:#8b949e; text-align:center; padding:20px;">No datasets archived yet.</div>';
+            return;
+        }
+
+        document.getElementById('hfList').innerHTML = folders.map(f => {
+            const cleanName = f.replace('Market Data/', '');
+            return `<div class="hf-item">
+                <b style="color:var(--accent)">📁 ${cleanName}</b>
+                <div class="hf-btns">
+                    <button class="hf-btn" onclick="App.viewMeta('${f}')">📝 Meta</button>
+                    <button class="hf-btn" onclick="App.viewData('${f}')">📊 View Data</button>
+                </div>
+            </div>`;
+        }).join('');
+    },
+
+    async viewMeta(folder) {
+        document.getElementById('metaBox').style.display = 'block';
+        document.getElementById('metaContent').innerText = 'Loading metadata...';
+        
+        const res = await fetch(`/api/hf_metadata?folder_path=${encodeURIComponent(folder)}`);
+        const data = await res.json();
+        
+        if(data.status === 'error') {
+            document.getElementById('metaContent').innerText = data.message;
+        } else {
+            document.getElementById('metaContent').innerText = JSON.stringify(data.metadata, null, 2);
+        }
+    },
+
+    async viewData(folder) {
+        document.getElementById('dataModal').style.display = 'flex';
+        document.getElementById('modalTitle').innerText = "Dataset: " + folder.replace('Market Data/', '');
+        const body = document.getElementById('modalBody');
+        body.innerHTML = '<div style="color:#8b949e; text-align:center; margin-top:50px;">Fetching and parsing Parquet file from Hugging Face...</div>';
+
+        const res = await fetch(`/api/view_parquet?folder_path=${encodeURIComponent(folder)}`);
+        const resp = await res.json();
+
+        if (resp.status === 'error') {
+            body.innerHTML = `<div style="color:var(--danger); text-align:center; margin-top:50px;">${resp.message}</div>`;
+            return;
+        }
+
+        let html = '<table class="data-table"><thead><tr>';
+        resp.columns.forEach(c => html += `<th>${c}</th>`);
+        html += '</tr></thead><tbody>';
+
+        resp.data.forEach(row => {
+            html += '<tr>';
+            resp.columns.forEach(c => {
+                let val = row[c];
+                if(typeof val === 'number') val = val.toFixed(2);
+                html += `<td>${val}</td>`;
+            });
+            html += '</tr>';
+        });
+        html += '</tbody></table>';
+        body.innerHTML = html;
+    }
+};
+
+window.onload = () => App.init();
+</script>
+</body>
+</html>
+"""
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=7860)