Update fetch script to upload parquet dataset to HuggingFace

- Add datasets and pyarrow dependencies to UV script
- Flatten nested JSON structure for parquet compatibility
- Create columns for all benchmarks (even if null)
- Upload directly to OpenEvals/leaderboard-data dataset
- Remove local file read/write dependencies
- Add aggregate score and coverage metrics
- 82 models, 11 benchmarks, 23 columns total

scripts/fetch_api_only.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # /// script
-# dependencies = ["requests", "huggingface-hub"]
+# dependencies = ["requests", "huggingface-hub", "datasets", "pyarrow"]
 # ///
 
 import requests
@@ -198,83 +198,147 @@ def fetch_all_from_apis(hf_token=None):
 
         print(f"  ✓ Found {len([e for e in data if e.get('modelId')])} models")
 
-    breakpoint()
     # Calculate aggregate scores
     models = list(models_dict.values())
-    for model in models:
-        benchmarks = model.get("benchmarks", {})
-        if benchmarks:
-            scores = [b["score"] for b in benchmarks.values()]
-            model["aggregateScore"] = round(sum(scores) / len(scores), 2)
-            model["coverageCount"] = len(benchmarks)
-            model["coveragePercent"] = round((len(benchmarks) / 12) * 100, 1)
-
-    # Sort by aggregate score
-    models.sort(key=lambda x: x["aggregateScore"], reverse=True)
 
     return models
 
 
+def flatten_model_for_parquet(model, all_benchmark_keys):
+    """Flatten nested model structure for parquet compatibility.
+
+    Converts nested JSON structure into flat columns suitable for parquet format.
+    Each benchmark score becomes its own column.
+
+    Args:
+        model: Model dict with nested structure
+        all_benchmark_keys: List of all possible benchmark keys to ensure consistent schema
+    """
+    flat = {
+        "model_id": model["id"],
+        "model_name": model["name"],
+        "provider": model["provider"],
+        "model_type": model["type"],
+        "parameters_billions": model["metadata"].get("parametersInBillions"),
+        "license": model["metadata"].get("license", "Unknown"),
+        "context_window": model["metadata"].get("contextWindow", 0),
+        "modality": model["metadata"].get("modality", "text"),
+        "architecture": model["metadata"].get("architecture", "Transformer"),
+    }
+
+    # Add ALL benchmark columns (with None for missing values)
+    # This ensures consistent schema across all rows
+    benchmarks = model.get("benchmarks", {})
+    for bench_key in sorted(all_benchmark_keys):
+        if bench_key in benchmarks:
+            bench_data = benchmarks[bench_key]
+            flat[f"{bench_key}_score"] = bench_data.get("value")
+        else:
+            flat[f"{bench_key}_score"] = None
+
+    # Calculate aggregate metrics
+    if benchmarks:
+        scores = [
+            b.get("value") for b in benchmarks.values() if b.get("value") is not None
+        ]
+        if scores:
+            flat["aggregate_score"] = round(sum(scores) / len(scores), 2)
+            flat["coverage_count"] = len(benchmarks)
+            flat["coverage_percent"] = round((len(benchmarks) / 11) * 100, 1)
+        else:
+            flat["aggregate_score"] = None
+            flat["coverage_count"] = 0
+            flat["coverage_percent"] = 0.0
+    else:
+        flat["aggregate_score"] = None
+        flat["coverage_count"] = 0
+        flat["coverage_percent"] = 0.0
+
+    return flat
+
+
 def main():
     print("=" * 70)
-    print("Fetching ONLY from Official APIs (No Manual Data)")
+    print("Fetching from Official APIs & Uploading to HF Dataset")
     print("=" * 70)
     print()
 
-    # Get HF token from environment or command line
-    hf_token = None
-    if len(sys.argv) > 1:
-        if sys.argv[1] == "--token" and len(sys.argv) > 2:
-            hf_token = sys.argv[2]
-            print("✓ Using token from command line")
+    # Get HF token from environment (required for upload)
+    hf_token = os.environ.get("HF_TOKEN")
 
     if not hf_token:
-        hf_token = os.environ.get("HF_TOKEN")
-        if hf_token:
-            print("✓ Using token from HF_TOKEN environment variable")
-
-    if hf_token:
-        print("🔓 Token provided - will attempt to fetch gated datasets (GPQA, HLE)")
-    else:
-        print("⚠️  No token provided - gated datasets will be skipped")
-        print("   To access gated datasets, use: --token YOUR_HF_TOKEN")
-        print("   or set HF_TOKEN environment variable")
+        print("❌ HF_TOKEN environment variable required")
+        print("   Export your token: export HF_TOKEN=your_token")
+        sys.exit(1)
 
+    print("✓ HF_TOKEN found")
+    print("🔓 Will fetch gated datasets (GPQA, HLE)")
     print()
 
+    # Fetch models from APIs
     models = fetch_all_from_apis(hf_token)
 
-    # Load benchmark definitions
-    try:
-        with open("data/leaderboard.json", "r") as f:
-            leaderboard_data = json.load(f)
-    except:
-        print("Error loading leaderboard.json")
-        return 1
-
-    # Replace models with API-only data
-    leaderboard_data["models"] = models
-    leaderboard_data["metadata"]["totalModels"] = len(models)
-    leaderboard_data["metadata"]["lastUpdated"] = datetime.now().isoformat() + "Z"
-
-    # Save
-    with open("data/leaderboard.json", "w") as f:
-        json.dump(leaderboard_data, indent=2, fp=f)
+    if not models:
+        print("❌ No models fetched - exiting")
+        sys.exit(0)
 
     print()
     print("=" * 70)
-    print(f"✓ Loaded {len(models)} models from APIs only")
+    print(f"✓ Fetched {len(models)} models from APIs")
     print("=" * 70)
 
-    # Show coverage
-    benchmarks = [
+    # Collect all benchmark keys to ensure consistent schema
+    all_benchmark_keys = set()
+    for m in models:
+        all_benchmark_keys.update(m.get("benchmarks", {}).keys())
+
+    print(
+        f"\n🔍 Found {len(all_benchmark_keys)} unique benchmarks: {sorted(all_benchmark_keys)}"
+    )
+
+    # Flatten data for parquet (pass all_benchmark_keys for consistent schema)
+    print("\n📊 Flattening data for parquet format...")
+    flattened_models = [
+        flatten_model_for_parquet(m, all_benchmark_keys) for m in models
+    ]
+
+    # Create HF Dataset
+    from datasets import Dataset
+
+    dataset = Dataset.from_list(flattened_models)
+
+    print(f"   ✓ Created dataset with {len(dataset)} rows")
+    print(f"   ✓ Schema: {len(dataset.column_names)} columns")
+
+    # Upload to HuggingFace
+    DATASET_REPO = "OpenEvals/leaderboard-data"
+
+    print(f"\n📤 Uploading to {DATASET_REPO}...")
+
+    try:
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")
+
+        dataset.push_to_hub(
+            DATASET_REPO,
+            token=hf_token,
+            commit_message=f"Automated update: {timestamp}",
+        )
+
+        print(f"   ✅ Successfully uploaded!")
+        print(f"   🔗 View at: https://huggingface.co/datasets/{DATASET_REPO}")
+
+    except Exception as e:
+        print(f"   ❌ Upload failed: {e}")
+        sys.exit(1)
+
+    # Show summary
+    benchmark_keys = [
         "gsm8k",
         "mmluPro",
         "gpqa",
         "hle",
         "olmOcr",
         "sweVerified",
-        "arguana",
         "swePro",
         "aime2026",
         "terminalBench",
@@ -282,17 +346,19 @@ def main():
         "hmmt2026",
     ]
 
-    print("\nBenchmark Coverage:")
-    for bench in benchmarks:
-        count = sum(1 for m in models if bench in m.get("benchmarks", {}))
-        if count > 0:
-            print(f"  {bench:20s}: {count:2d} models")
-
-    print("\nTop 10 Models:")
-    for i, m in enumerate(models[:10], 1):
-        print(f"  {i:2d}. {m['name']:<40s} {m['aggregateScore']:>5.1f}")
-
-    print("\n✓ Data updated - 100% from APIs!")
+    print("\n📊 Benchmark Coverage:")
+    for bench in benchmark_keys:
+        col_name = f"{bench}_score"
+        if col_name in dataset.column_names:
+            # Count non-null values in the column
+            values = dataset[col_name]
+            count = sum(1 for v in values if v is not None)
+            if count > 0:
+                print(f"   {bench:20s}: {count:2d} models")
+
+    print("\n✅ Data updated successfully!")
+    print(f"   Total models: {len(models)}")
+    print(f"   Timestamp: {timestamp}")
 
 
 if __name__ == "__main__":