Upload app.py with huggingface_hub

app.py

@@ -0,0 +1,152 @@
+import gradio as gr
+import duckdb
+import pandas as pd
+import json
+import os
+from huggingface_hub import HfApi, hf_hub_download
+import time
+
+# --- Configuration ---
+SOURCE_REPO = "SII-WANGZJ/Polymarket_data"
+DEST_REPO = "Vycka12/Base" # User's target dataset
+TRADES_FILE = "trades.parquet"
+MARKETS_FILE = "markets.parquet"
+
+def run_extraction():
+    try:
+        token = os.environ.get("HF_TOKEN")
+        if not token:
+            return "❌ Missing HF_TOKEN! Please add it to Space Secrets (Settings -> Secrets)."
+
+        api = HfApi()
+        
+        # 1. Initialize DuckDB with HTTP extension
+        print("Initializing DuckDB with HTTPFS...")
+        con = duckdb.connect(database=':memory:')
+        con.execute("INSTALL httpfs; LOAD httpfs;")
+        
+        # Get direct download URLs (requires token)
+        trades_url = f"https://huggingface.co/datasets/{SOURCE_REPO}/resolve/main/{TRADES_FILE}"
+        markets_url = f"https://huggingface.co/datasets/{SOURCE_REPO}/resolve/main/{MARKETS_FILE}"
+        
+        # 2. Identify winning assets (Reading markets.parquet remotely)
+        print("Mapping winning assets from cloud...")
+        winning_mapping_query = f"""
+        CREATE TABLE winning_assets AS 
+        WITH cleaned_markets AS (
+            SELECT 
+                token1, token2, 
+                CAST(replace(replace(outcome_prices, '''', '"'), ' ', '') AS DOUBLE[]) as prices
+            FROM read_parquet('{markets_url}')
+            WHERE closed = 1 AND outcome_prices IS NOT NULL AND outcome_prices != ''
+        )
+        SELECT token1 as asset_id FROM cleaned_markets WHERE prices[1] = 1.0
+        UNION
+        SELECT token2 as asset_id FROM cleaned_markets WHERE prices[2] = 1.0
+        """
+        con.execute(winning_mapping_query)
+        
+        losing_mapping_query = """
+        CREATE TABLE losing_assets AS 
+        SELECT token2 as asset_id FROM winning_assets -- Simplify for this batch
+        UNION
+        SELECT token1 as asset_id FROM winning_assets
+        """
+        # (Better to just reuse logic but let's keep it simple for now)
+        
+        # 3. Analyze trades (32GB JOIN remotely over HTTP)
+        # DuckDB will only fetch the needed columns and rows via range requests
+        print("Calculating wallet stats from 32GB trades cloud file...")
+        stats_query = f"""
+        CREATE TABLE wallet_stats AS
+        WITH all_trades AS (
+            SELECT maker as wallet, asset_id FROM read_parquet('{trades_url}') WHERE maker_direction = 'BUY'
+            UNION ALL
+            SELECT taker as wallet, asset_id FROM read_parquet('{trades_url}') WHERE taker_direction = 'BUY'
+        ),
+        wins AS (
+            SELECT wallet, count(*) as win_count FROM all_trades 
+            WHERE asset_id IN (SELECT asset_id FROM winning_assets)
+            GROUP BY wallet
+        ),
+        losses AS (
+            SELECT wallet, count(*) as loss_count FROM all_trades 
+            WHERE asset_id IN (SELECT asset_id FROM (
+                SELECT token2 as asset_id FROM (
+                    SELECT token1, token2, CAST(replace(replace(outcome_prices, '''', '"'), ' ', '') AS DOUBLE[]) as prices
+                    FROM read_parquet('{markets_url}')
+                    WHERE closed = 1 AND (token1 IS NOT NULL OR token2 IS NOT NULL)
+                ) WHERE prices[1] = 1.0
+                UNION
+                SELECT token1 as asset_id FROM (
+                     SELECT token1, token2, CAST(replace(replace(outcome_prices, '''', '"'), ' ', '') AS DOUBLE[]) as prices
+                    FROM read_parquet('{markets_url}')
+                    WHERE closed = 1
+                ) WHERE prices[2] = 1.0
+            ))
+            GROUP BY wallet
+        )
+        SELECT 
+            COALESCE(w.wallet, l.wallet) as wallet,
+            COALESCE(w.win_count, 0) as wins,
+            COALESCE(l.loss_count, 0) as losses,
+            (COALESCE(w.win_count, 0) + COALESCE(l.loss_count, 0)) as total_resolved_trades,
+            CAST(COALESCE(w.win_count, 0) AS DOUBLE) / (COALESCE(w.win_count, 0) + COALESCE(l.loss_count, 0)) as win_rate
+        FROM wins w
+        FULL OUTER JOIN losses l ON w.wallet = l.wallet
+        WHERE (COALESCE(w.win_count, 0) + COALESCE(l.loss_count, 0)) >= 100
+        """
+        con.execute(stats_query)
+        
+        # 4. Filter Results
+        print("Filtering final results...")
+        df = con.execute("SELECT * FROM wallet_stats").df()
+        
+        whales = df[(df['total_resolved_trades'] >= 400) & (df['win_rate'] >= 0.70)].to_dict('records')
+        dumb = df[(df['total_resolved_trades'] >= 100) & (df['win_rate'] <= 0.40)].to_dict('records')
+        
+        # 5. Save and Upload to HF
+        os.makedirs("results", exist_ok=True)
+        whale_path = "results/mega_whales.json"
+        dumb_path = "results/mega_dumb.json"
+        
+        with open(whale_path, "w") as f:
+            json.dump(whales, f, indent=2)
+        with open(dumb_path, "w") as f:
+            json.dump(dumb, f, indent=2)
+            
+        print(f"Uploading Results to {DEST_REPO}...")
+        api.upload_file(
+            path_or_fileobj=whale_path,
+            path_in_repo="results/mega_whales.json",
+            repo_id=DEST_REPO,
+            repo_type="dataset",
+            token=token
+        )
+        api.upload_file(
+            path_or_fileobj=dumb_path,
+            path_in_repo="results/mega_dumb.json",
+            repo_id=DEST_REPO,
+            repo_type="dataset",
+            token=token
+        )
+        
+        return f"✅ SUCCESS!\n- Found {len(whales)} Mega Whales\n- Found {len(dumb)} Mega Dumb\n- Results uploaded to {DEST_REPO}/results/"
+        
+    except Exception as e:
+        return f"❌ ERROR: {str(e)}"
+
+# --- Gradio UI ---
+with gr.Blocks(title="Mega Extraction Space") as demo:
+    gr.Markdown("# 🐋 Mega Wallet Extractor (DuckDB Cloud)")
+    gr.Markdown("Click the button below to process 32GB of Polymarket data directly from Hugging Face.")
+    
+    with gr.Row():
+        run_btn = gr.Button("🚀 Run Full Extraction", variant="primary")
+    
+    status_box = gr.Textbox(label="Status Log", lines=10)
+    
+    run_btn.click(fn=run_extraction, outputs=status_box)
+
+if __name__ == "__main__":
+    demo.launch()