| { |
| "benchmark_name": "NullspanBench (JevBench-Compatible Decision & Extraction Leaderboard)", |
| "benchmark_version": "3.2.0-jevbench", |
| "draw_id": "S+P (draw v3.2-nullspan-20261030)", |
| "timestamp": "2026-10-30T13:11:15Z", |
| "n_total_observations": 500, |
| "n_open_observations": 336, |
| "n_sealed_observations": 164, |
| "environment": { |
| "os": "Linux 6.8.0-1070-gcp (x86_64)", |
| "python": "3.10.12", |
| "pytorch": "2.9.1+cu129", |
| "onnxruntime": "1.23.2", |
| "accelerator": "NVIDIA L4 (GCP CUDA Eval)" |
| }, |
| "systems": [ |
| { |
| "id": "nullspan-hybrid-router", |
| "name": "nullspan-Hybrid (Fast + ONNX Int8)", |
| "author": "gnumanth", |
| "system_type": "Hybrid System One Router", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 331.1, |
| "base_model": "Zero-Dep Router + gnumanth/nullspan Int8", |
| "runtime": "Tiered JS/Python + ONNX Runtime", |
| "input_rate_per_1m_usd": 0.002, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0003, |
| "axes": { |
| "intelligence": 97.8, |
| "intelligence_ci95": [ |
| 96.6, |
| 99.0 |
| ], |
| "calibration": 98.6, |
| "speed": 92.7, |
| "cost": 93.8 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 95.7, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.3, |
| "calibration_details": { |
| "null_auroc": 0.99629, |
| "null_ece": 0.01392, |
| "null_brier": 0.01516 |
| }, |
| "latency": { |
| "mean_ms": 16.8098, |
| "p50_ms": 0.0397, |
| "p95_ms": 126.6229, |
| "p99_ms": 209.086 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 84.1, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 86.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 81.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 1 |
| }, |
| { |
| "id": "nullspan-zerodep-fast", |
| "name": "nullspan-ZeroDep (Fast Engine)", |
| "author": "gnumanth", |
| "system_type": "Zero-Dep Algorithmic Engine", |
| "open_weights": true, |
| "params_m": 0.0, |
| "footprint_mb": 0.0, |
| "base_model": "Deterministic Clause + Lexical Pointer", |
| "runtime": "Pure Python / Node / Browser JS", |
| "input_rate_per_1m_usd": 0.0008, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0001, |
| "axes": { |
| "intelligence": 90.8, |
| "intelligence_ci95": [ |
| 88.2, |
| 93.4 |
| ], |
| "calibration": 95.0, |
| "speed": 93.5, |
| "cost": 99.8 |
| }, |
| "capability_score": 92.9, |
| "composite_score": 94.7, |
| "open_set_score": 100.0, |
| "sealed_set_score": 72.0, |
| "calibration_details": { |
| "null_auroc": 0.98239, |
| "null_ece": 0.04866, |
| "null_brier": 0.05142 |
| }, |
| "latency": { |
| "mean_ms": 0.1345, |
| "p50_ms": 0.0342, |
| "p95_ms": 0.6895, |
| "p99_ms": 0.8689 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 78.7, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 47.7, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 79.6, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 54.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 23.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 2 |
| }, |
| { |
| "id": "nullspan-fp32", |
| "name": "nullspan-345M (PyTorch FP32)", |
| "author": "gnumanth", |
| "system_type": "System One Encoder (5-Head, 512-ctx)", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 1317.2, |
| "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", |
| "runtime": "PyTorch MPS/CUDA/CPU FP32", |
| "input_rate_per_1m_usd": 0.012, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0017, |
| "axes": { |
| "intelligence": 98.0, |
| "intelligence_ci95": [ |
| 96.8, |
| 99.2 |
| ], |
| "calibration": 98.5, |
| "speed": 63.2, |
| "cost": 76.8 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 81.3, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.9, |
| "calibration_details": { |
| "null_auroc": 0.99327, |
| "null_ece": 0.01419, |
| "null_brier": 0.01532 |
| }, |
| "latency": { |
| "mean_ms": 16.9035, |
| "p50_ms": 10.2968, |
| "p95_ms": 33.0811, |
| "p99_ms": 37.3921 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 86.4, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 88.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 83.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 3 |
| }, |
| { |
| "id": "nullspan-onnx-int8", |
| "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", |
| "author": "gnumanth", |
| "system_type": "Quantized ONNX WASM/CPU (512-ctx)", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 331.1, |
| "base_model": "gnumanth/nullspan (Int8 Dynamic)", |
| "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", |
| "input_rate_per_1m_usd": 0.006, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0008, |
| "axes": { |
| "intelligence": 97.8, |
| "intelligence_ci95": [ |
| 96.6, |
| 99.0 |
| ], |
| "calibration": 98.6, |
| "speed": 52.7, |
| "cost": 84.2 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 78.1, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.3, |
| "calibration_details": { |
| "null_auroc": 0.99511, |
| "null_ece": 0.01267, |
| "null_brier": 0.01523 |
| }, |
| "latency": { |
| "mean_ms": 108.0495, |
| "p50_ms": 75.9329, |
| "p95_ms": 207.004, |
| "p99_ms": 369.9251 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 84.1, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 86.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 81.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 4 |
| }, |
| { |
| "id": "h2o-lightning-4b", |
| "name": "H2O-Lightning-4B v1.1 (JevBench #1 Comp)", |
| "author": "H2O.ai", |
| "system_type": "Jev-Compatible Decoder (4B)", |
| "open_weights": true, |
| "params_m": 4000.0, |
| "footprint_mb": 8100.0, |
| "base_model": "Qwen/Qwen3.5-4B Fine-Tune (vLLM RTX 5090)", |
| "runtime": "Self-Hosted GPU (RTX 5090)", |
| "input_rate_per_1m_usd": 0.027, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 1500, |
| "input_tokens_total": 1152000, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0211, |
| "axes": { |
| "intelligence": 60.0, |
| "intelligence_ci95": [ |
| 57.8, |
| 62.3 |
| ], |
| "calibration": 90.0, |
| "speed": 92.6, |
| "cost": 60.3 |
| }, |
| "capability_score": 75.0, |
| "composite_score": 72.5, |
| "open_set_score": 64.1, |
| "sealed_set_score": 59.0, |
| "calibration_details": { |
| "null_auroc": 0.908, |
| "null_ece": 0.0502, |
| "null_brier": 0.0645 |
| }, |
| "latency": { |
| "mean_ms": 34.1, |
| "p50_ms": 29.285, |
| "p95_ms": 58.4, |
| "p99_ms": 74.0 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 61.5, |
| "n": 300 |
| }, |
| "span_sealed": { |
| "score": 58.2, |
| "n": 300 |
| }, |
| "noul_open": { |
| "score": 60.9, |
| "n": 150 |
| }, |
| "noul_sealed": { |
| "score": 68.1, |
| "n": 150 |
| }, |
| "choice": { |
| "score": 75.2, |
| "n": 300 |
| }, |
| "math": { |
| "score": 57.6, |
| "n": 150 |
| }, |
| "score": { |
| "score": 47.0, |
| "n": 150 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 88.2, |
| "n": 100 |
| }, |
| "Standard": { |
| "score": 57.6, |
| "n": 100 |
| }, |
| "Judge": { |
| "score": 61.0, |
| "n": 100 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 62.6, |
| "n": 600 |
| }, |
| "Hard": { |
| "score": 54.3, |
| "n": 600 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 69.4, |
| "n": 300 |
| }, |
| "compute_ledger": { |
| "score": 59.2, |
| "n": 300 |
| }, |
| "cluster_throughput_comps": { |
| "score": 47.0, |
| "n": 150 |
| }, |
| "edge_traffic_forecast": { |
| "score": 44.3, |
| "n": 150 |
| }, |
| "squad_v2_open_qa": { |
| "score": 62.6, |
| "n": 300 |
| }, |
| "contract_extraction": { |
| "score": 58.4, |
| "n": 300 |
| }, |
| "adversarial_stress": { |
| "score": 54.3, |
| "n": 300 |
| } |
| }, |
| "official_rank": 5 |
| }, |
| { |
| "id": "cloudflare-clairic-clef-live", |
| "name": "@cf/cloudflare/clairic + clef (Live Workers AI)", |
| "author": "Cloudflare", |
| "system_type": "System One Split API (Clairic + Clef 27B)", |
| "open_weights": false, |
| "params_m": 27000.0, |
| "footprint_mb": null, |
| "base_model": "@cf/cloudflare/clairic + @cf/cloudflare/clef (180 RPCs / 246 Passes)", |
| "runtime": "Cloudflare Workers AI Edge RPC (p50 315.3 ms/doc)", |
| "input_rate_per_1m_usd": 0.025, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 81206, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0041, |
| "axes": { |
| "intelligence": 91.6, |
| "intelligence_ci95": [ |
| 89.2, |
| 93.9 |
| ], |
| "calibration": 94.9, |
| "speed": 51.7, |
| "cost": 68.2 |
| }, |
| "capability_score": 93.2, |
| "composite_score": 72.1, |
| "open_set_score": 89.7, |
| "sealed_set_score": 95.4, |
| "calibration_details": { |
| "null_auroc": 0.98959, |
| "null_ece": 0.05235, |
| "null_brier": 0.04728 |
| }, |
| "latency": { |
| "mean_ms": 128.789, |
| "p50_ms": 91.323, |
| "p95_ms": 314.023, |
| "p99_ms": 403.1387 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 83.8, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 89.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 95.8, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 93.2, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 85.8, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 94.5, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 93.4, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 92.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 94.4, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 82.6, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 96.2, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 92.5, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 97.9, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 87.5, |
| "n": 8 |
| } |
| }, |
| "official_rank": 6 |
| }, |
| { |
| "id": "basal-omni-1.0-ultra", |
| "name": "basal-omni-1.0-ultra (JevBench #1 Cap)", |
| "author": "Remek Kinas", |
| "system_type": "Jev-Compatible Decoder (31B)", |
| "open_weights": true, |
| "params_m": 31000.0, |
| "footprint_mb": 62000.0, |
| "base_model": "Gemma-4-31B-it + Merged LoRA (Typed Readout)", |
| "runtime": "Self-Hosted GPU (p50 102 ms)", |
| "input_rate_per_1m_usd": 0.058, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 1500, |
| "input_tokens_total": 1152000, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0447, |
| "axes": { |
| "intelligence": 74.1, |
| "intelligence_ci95": [ |
| 71.8, |
| 76.3 |
| ], |
| "calibration": 91.0, |
| "speed": 87.3, |
| "cost": 50.5 |
| }, |
| "capability_score": 82.5, |
| "composite_score": 71.8, |
| "open_set_score": 76.7, |
| "sealed_set_score": 73.1, |
| "calibration_details": { |
| "null_auroc": 0.924, |
| "null_ece": 0.0449, |
| "null_brier": 0.055 |
| }, |
| "latency": { |
| "mean_ms": 114.2, |
| "p50_ms": 102.0, |
| "p95_ms": 168.0, |
| "p99_ms": 210.0 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 75.4, |
| "n": 300 |
| }, |
| "span_sealed": { |
| "score": 73.8, |
| "n": 300 |
| }, |
| "noul_open": { |
| "score": 68.4, |
| "n": 150 |
| }, |
| "noul_sealed": { |
| "score": 72.5, |
| "n": 150 |
| }, |
| "choice": { |
| "score": 82.7, |
| "n": 300 |
| }, |
| "math": { |
| "score": 69.1, |
| "n": 150 |
| }, |
| "score": { |
| "score": 69.1, |
| "n": 150 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 96.5, |
| "n": 100 |
| }, |
| "Standard": { |
| "score": 69.1, |
| "n": 100 |
| }, |
| "Judge": { |
| "score": 65.2, |
| "n": 100 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 78.8, |
| "n": 600 |
| }, |
| "Hard": { |
| "score": 74.0, |
| "n": 600 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 81.2, |
| "n": 300 |
| }, |
| "compute_ledger": { |
| "score": 71.5, |
| "n": 300 |
| }, |
| "cluster_throughput_comps": { |
| "score": 69.1, |
| "n": 300 |
| }, |
| "edge_traffic_forecast": { |
| "score": 66.9, |
| "n": 150 |
| }, |
| "squad_v2_open_qa": { |
| "score": 76.9, |
| "n": 300 |
| }, |
| "contract_extraction": { |
| "score": 72.8, |
| "n": 300 |
| }, |
| "adversarial_stress": { |
| "score": 74.0, |
| "n": 300 |
| } |
| }, |
| "official_rank": 7 |
| }, |
| { |
| "id": "jev-1.13.0-ref", |
| "name": "Jev 1.13.0 (TypeSafe AI \u00b7 JevBench Ref)", |
| "author": "TypeSafe AI", |
| "system_type": "Jev Reference \u00b7 Hosted API", |
| "open_weights": false, |
| "params_m": null, |
| "footprint_mb": null, |
| "base_model": "TypeSafe Jev 1.13.0 ($0.042/1M input)", |
| "runtime": "TypeSafe Hosted API (JevBench v1.7)", |
| "input_rate_per_1m_usd": 0.042, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 1500, |
| "input_tokens_total": 1152000, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0323, |
| "axes": { |
| "intelligence": 63.6, |
| "intelligence_ci95": [ |
| 61.2, |
| 66.0 |
| ], |
| "calibration": 90.6, |
| "speed": 91.5, |
| "cost": 54.7 |
| }, |
| "capability_score": 77.1, |
| "composite_score": 71.5, |
| "open_set_score": 68.9, |
| "sealed_set_score": 61.6, |
| "calibration_details": { |
| "null_auroc": 0.912, |
| "null_ece": 0.0468, |
| "null_brier": 0.0612 |
| }, |
| "latency": { |
| "mean_ms": 251.4, |
| "p50_ms": 239.102, |
| "p95_ms": 412.0, |
| "p99_ms": 530.0 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 66.2, |
| "n": 300 |
| }, |
| "span_sealed": { |
| "score": 61.4, |
| "n": 300 |
| }, |
| "noul_open": { |
| "score": 54.5, |
| "n": 150 |
| }, |
| "noul_sealed": { |
| "score": 46.1, |
| "n": 150 |
| }, |
| "choice": { |
| "score": 79.0, |
| "n": 300 |
| }, |
| "math": { |
| "score": 63.2, |
| "n": 150 |
| }, |
| "score": { |
| "score": 63.3, |
| "n": 150 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 93.9, |
| "n": 100 |
| }, |
| "Standard": { |
| "score": 63.2, |
| "n": 100 |
| }, |
| "Judge": { |
| "score": 67.7, |
| "n": 100 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 68.7, |
| "n": 600 |
| }, |
| "Hard": { |
| "score": 58.9, |
| "n": 600 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 74.2, |
| "n": 300 |
| }, |
| "compute_ledger": { |
| "score": 66.8, |
| "n": 300 |
| }, |
| "cluster_throughput_comps": { |
| "score": 63.3, |
| "n": 150 |
| }, |
| "edge_traffic_forecast": { |
| "score": 63.0, |
| "n": 150 |
| }, |
| "squad_v2_open_qa": { |
| "score": 61.6, |
| "n": 300 |
| }, |
| "contract_extraction": { |
| "score": 62.4, |
| "n": 300 |
| }, |
| "adversarial_stress": { |
| "score": 58.9, |
| "n": 300 |
| } |
| }, |
| "official_rank": 8 |
| }, |
| { |
| "id": "nullspan-onnx-fp32", |
| "name": "nullspan-345M (ONNX FP32)", |
| "author": "gnumanth", |
| "system_type": "System One Encoder (ONNX, 512-ctx)", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 1317.6, |
| "base_model": "gnumanth/nullspan (ONNX Opset 17)", |
| "runtime": "ONNX Runtime FP32", |
| "input_rate_per_1m_usd": 0.011, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0015, |
| "axes": { |
| "intelligence": 98.0, |
| "intelligence_ci95": [ |
| 96.8, |
| 99.2 |
| ], |
| "calibration": 98.5, |
| "speed": 43.5, |
| "cost": 78.0 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 71.2, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.9, |
| "calibration_details": { |
| "null_auroc": 0.99327, |
| "null_ece": 0.01419, |
| "null_brier": 0.01532 |
| }, |
| "latency": { |
| "mean_ms": 491.2899, |
| "p50_ms": 424.9432, |
| "p95_ms": 1074.4535, |
| "p99_ms": 1256.8557 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 86.4, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 88.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 83.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 9 |
| }, |
| { |
| "id": "deberta-v3-base-squad2", |
| "name": "deberta-v3-base-squad2 (184M SQuAD2 QA)", |
| "author": "deepset", |
| "system_type": "Open Weights \u00b7 Extractive QA Cross-Encoder", |
| "open_weights": true, |
| "params_m": 183.8, |
| "footprint_mb": 701.3, |
| "base_model": "deepset/deberta-v3-base-squad2", |
| "runtime": "PyTorch MPS/CPU (512-ctx)", |
| "input_rate_per_1m_usd": 0.032, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 57964, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0037, |
| "axes": { |
| "intelligence": 48.1, |
| "intelligence_ci95": [ |
| 43.9, |
| 52.4 |
| ], |
| "calibration": 63.4, |
| "speed": 65.1, |
| "cost": 69.2 |
| }, |
| "capability_score": 55.8, |
| "composite_score": 60.3, |
| "open_set_score": 48.6, |
| "sealed_set_score": 47.3, |
| "calibration_details": { |
| "null_auroc": 0.82087, |
| "null_ece": 0.39493, |
| "null_brier": 0.31346 |
| }, |
| "latency": { |
| "mean_ms": 9.6505, |
| "p50_ms": 7.2341, |
| "p95_ms": 24.8532, |
| "p99_ms": 26.2559 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 56.4, |
| "n": 181 |
| }, |
| "math": { |
| "score": 4.9, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 65.9, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 15.1, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 35.2, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 84.1, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 64.2, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 9.6, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 64.1, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 33.6, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 78.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 50.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 50.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 57.6, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 1.3, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 80.0, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 27.6, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 37.5, |
| "n": 8 |
| } |
| }, |
| "official_rank": 10 |
| }, |
| { |
| "id": "gpt-5.4-nano-json", |
| "name": "gpt-5.4-nano (JSON Decoder \u00b7 API)", |
| "author": "OpenAI (Ref)", |
| "system_type": "Closed API \u00b7 LLM Decoder", |
| "open_weights": false, |
| "params_m": null, |
| "footprint_mb": null, |
| "base_model": "Autoregressive JSON Schema Decoder", |
| "runtime": "Hosted Cloud API ( ~38 output tokens/field )", |
| "input_rate_per_1m_usd": 0.15, |
| "output_rate_per_1m_usd": 0.6, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 19000, |
| "usd_per_1k_decisions": 0.0438, |
| "axes": { |
| "intelligence": 95.2, |
| "intelligence_ci95": [ |
| 93.4, |
| 97.0 |
| ], |
| "calibration": 89.0, |
| "speed": 43.6, |
| "cost": 45.1 |
| }, |
| "capability_score": 92.1, |
| "composite_score": 59.8, |
| "open_set_score": 99.7, |
| "sealed_set_score": 86.0, |
| "calibration_details": { |
| "null_auroc": 0.95791, |
| "null_ece": 0.15112, |
| "null_brier": 0.03403 |
| }, |
| "latency": { |
| "mean_ms": 420.4635, |
| "p50_ms": 417.9908, |
| "p95_ms": 515.593, |
| "p99_ms": 524.2688 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 96.3, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 91.7, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 68.2, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 99.4, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 92.0, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 72.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 99.4, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 76.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 91.7, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 87.5, |
| "n": 8 |
| } |
| }, |
| "official_rank": 11 |
| }, |
| { |
| "id": "claude-haiku-4.5-json", |
| "name": "claude-haiku-4.5 (JSON Decoder \u00b7 API)", |
| "author": "Anthropic (Ref)", |
| "system_type": "Closed API \u00b7 LLM Decoder", |
| "open_weights": false, |
| "params_m": null, |
| "footprint_mb": null, |
| "base_model": "Autoregressive JSON Schema Decoder", |
| "runtime": "Hosted Cloud API ( ~38 output tokens/field )", |
| "input_rate_per_1m_usd": 0.8, |
| "output_rate_per_1m_usd": 4.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 19000, |
| "usd_per_1k_decisions": 0.2639, |
| "axes": { |
| "intelligence": 95.4, |
| "intelligence_ci95": [ |
| 93.4, |
| 97.2 |
| ], |
| "calibration": 88.7, |
| "speed": 41.8, |
| "cost": 27.5 |
| }, |
| "capability_score": 92.1, |
| "composite_score": 48.8, |
| "open_set_score": 99.4, |
| "sealed_set_score": 87.2, |
| "calibration_details": { |
| "null_auroc": 0.98592, |
| "null_ece": 0.1566, |
| "null_brier": 0.03222 |
| }, |
| "latency": { |
| "mean_ms": 585.679, |
| "p50_ms": 586.3918, |
| "p95_ms": 717.8024, |
| "p99_ms": 727.6919 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 92.6, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 92.6, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 70.5, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 98.9, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 92.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 74.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 99.4, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 99.3, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 78.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 93.8, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 75.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 12 |
| }, |
| { |
| "id": "bge-small-en-v1.5", |
| "name": "bge-small-en-v1.5 (Dense Span+Null)", |
| "author": "BAAI", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 33.4, |
| "footprint_mb": 127.3, |
| "base_model": "BAAI/bge-small-en-v1.5", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.01, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0007, |
| "axes": { |
| "intelligence": 14.7, |
| "intelligence_ci95": [ |
| 12.0, |
| 17.5 |
| ], |
| "calibration": 89.2, |
| "speed": 64.2, |
| "cost": 85.5 |
| }, |
| "capability_score": 52.0, |
| "composite_score": 37.6, |
| "open_set_score": 20.3, |
| "sealed_set_score": 3.4, |
| "calibration_details": { |
| "null_auroc": 0.68694, |
| "null_ece": 0.10548, |
| "null_brier": 0.11334 |
| }, |
| "latency": { |
| "mean_ms": 8.6668, |
| "p50_ms": 8.6253, |
| "p95_ms": 8.9805, |
| "p99_ms": 9.3022 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 12.2, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 59.8, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 44.4, |
| "n": 27 |
| }, |
| "score": { |
| "score": 24.4, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 2.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 2.3, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 19.3, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 12.2, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 35.2, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 2.7, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 5.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 29.9, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 9.7, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 20.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 20.1, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 0.0, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 3.1, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 31.2, |
| "n": 8 |
| } |
| }, |
| "official_rank": 13 |
| }, |
| { |
| "id": "distilbert-base-control", |
| "name": "distilbert-base-control (Raw Control)", |
| "author": "distilbert (control)", |
| "system_type": "Base Model Control (Untuned)", |
| "open_weights": true, |
| "params_m": 66.4, |
| "footprint_mb": 253.2, |
| "base_model": "distilbert/distilbert-base-uncased", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.015, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0011, |
| "axes": { |
| "intelligence": 14.2, |
| "intelligence_ci95": [ |
| 11.6, |
| 17.0 |
| ], |
| "calibration": 91.5, |
| "speed": 67.7, |
| "cost": 81.1 |
| }, |
| "capability_score": 52.9, |
| "composite_score": 36.9, |
| "open_set_score": 20.2, |
| "sealed_set_score": 1.8, |
| "calibration_details": { |
| "null_auroc": 0.60728, |
| "null_ece": 0.07034, |
| "null_brier": 0.11186 |
| }, |
| "latency": { |
| "mean_ms": 4.552, |
| "p50_ms": 4.4787, |
| "p95_ms": 5.4739, |
| "p99_ms": 5.5359 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 16.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 59.8, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 0.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 34.2, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 0.9, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 0.0, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 16.5, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 17.1, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 35.2, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 1.8, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 2.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 24.2, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 11.1, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 27.4, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 55.1, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 1.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 1.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 12.5, |
| "n": 8 |
| } |
| }, |
| "official_rank": 14 |
| }, |
| { |
| "id": "all-MiniLM-L12-v2", |
| "name": "all-MiniLM-L12-v2 (Dense Span+Null)", |
| "author": "sentence-transformers", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 33.4, |
| "footprint_mb": 127.3, |
| "base_model": "sentence-transformers/all-MiniLM-L12-v2", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.01, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0007, |
| "axes": { |
| "intelligence": 14.4, |
| "intelligence_ci95": [ |
| 11.7, |
| 17.5 |
| ], |
| "calibration": 73.5, |
| "speed": 64.2, |
| "cost": 85.5 |
| }, |
| "capability_score": 44.0, |
| "composite_score": 36.3, |
| "open_set_score": 17.2, |
| "sealed_set_score": 8.8, |
| "calibration_details": { |
| "null_auroc": 0.58938, |
| "null_ece": 0.28318, |
| "null_brier": 0.23193 |
| }, |
| "latency": { |
| "mean_ms": 8.6507, |
| "p50_ms": 8.5815, |
| "p95_ms": 8.9115, |
| "p99_ms": 9.0401 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 9.4, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 28.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 6.7, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 0.5, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 29.5, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 25.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 3.3, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 16.4, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 0.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 27.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 19.1, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 16.7, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 20.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 1.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 11.5, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 31.2, |
| "n": 8 |
| } |
| }, |
| "official_rank": 15 |
| }, |
| { |
| "id": "all-mpnet-base-v2", |
| "name": "all-mpnet-base-v2 (Dense Span+Null)", |
| "author": "sentence-transformers", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 109.5, |
| "footprint_mb": 417.7, |
| "base_model": "sentence-transformers/all-mpnet-base-v2", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.022, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0016, |
| "axes": { |
| "intelligence": 13.6, |
| "intelligence_ci95": [ |
| 10.9, |
| 16.7 |
| ], |
| "calibration": 71.0, |
| "speed": 63.3, |
| "cost": 77.4 |
| }, |
| "capability_score": 42.3, |
| "composite_score": 34.4, |
| "open_set_score": 16.2, |
| "sealed_set_score": 8.2, |
| "calibration_details": { |
| "null_auroc": 0.58731, |
| "null_ece": 0.30424, |
| "null_brier": 0.26407 |
| }, |
| "latency": { |
| "mean_ms": 10.2958, |
| "p50_ms": 10.2572, |
| "p95_ms": 10.5562, |
| "p99_ms": 12.7047 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 9.4, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 28.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 0.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 0.5, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 27.3, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 25.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 16.4, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 0.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 25.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 19.1, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 16.7, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 0.0, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 11.5, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 31.2, |
| "n": 8 |
| } |
| }, |
| "official_rank": 16 |
| }, |
| { |
| "id": "paraphrase-albert-small-v2", |
| "name": "paraphrase-albert-small-v2 (Dense Span+Null)", |
| "author": "sentence-transformers", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 11.7, |
| "footprint_mb": 44.6, |
| "base_model": "sentence-transformers/paraphrase-albert-small-v2", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.005, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0004, |
| "axes": { |
| "intelligence": 12.5, |
| "intelligence_ci95": [ |
| 10.0, |
| 15.2 |
| ], |
| "calibration": 76.6, |
| "speed": 66.4, |
| "cost": 91.0 |
| }, |
| "capability_score": 44.5, |
| "composite_score": 33.6, |
| "open_set_score": 15.3, |
| "sealed_set_score": 6.7, |
| "calibration_details": { |
| "null_auroc": 0.58482, |
| "null_ece": 0.24547, |
| "null_brier": 0.21391 |
| }, |
| "latency": { |
| "mean_ms": 5.7054, |
| "p50_ms": 5.7004, |
| "p95_ms": 5.8183, |
| "p99_ms": 5.8798 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 11.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 1.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 35.4, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 51.9, |
| "n": 27 |
| }, |
| "score": { |
| "score": 8.4, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 2.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 15.9, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 19.3, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 4.2, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 21.1, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 2.7, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 15.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 50.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 21.1, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 9.7, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 20.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 3.4, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 0.0, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 9.9, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 18.8, |
| "n": 8 |
| } |
| }, |
| "official_rank": 17 |
| }, |
| { |
| "id": "paraphrase-MiniLM-L3-v2", |
| "name": "paraphrase-MiniLM-L3-v2 (Dense Span+Null)", |
| "author": "sentence-transformers", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 17.4, |
| "footprint_mb": 66.3, |
| "base_model": "sentence-transformers/paraphrase-MiniLM-L3-v2", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.005, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0004, |
| "axes": { |
| "intelligence": 11.1, |
| "intelligence_ci95": [ |
| 8.7, |
| 13.8 |
| ], |
| "calibration": 72.3, |
| "speed": 69.6, |
| "cost": 91.0 |
| }, |
| "capability_score": 41.7, |
| "composite_score": 30.9, |
| "open_set_score": 12.9, |
| "sealed_set_score": 7.6, |
| "calibration_details": { |
| "null_auroc": 0.58932, |
| "null_ece": 0.29025, |
| "null_brier": 0.25176 |
| }, |
| "latency": { |
| "mean_ms": 3.1323, |
| "p50_ms": 3.1142, |
| "p95_ms": 3.2437, |
| "p99_ms": 3.2779 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 9.7, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 28.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 44.4, |
| "n": 27 |
| }, |
| "score": { |
| "score": 6.7, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 0.5, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 25.0, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 16.8, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 3.3, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 16.4, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 0.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 23.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 17.5, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 8.3, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 20.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 1.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 10.4, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 18.8, |
| "n": 8 |
| } |
| }, |
| "official_rank": 18 |
| }, |
| { |
| "id": "all-MiniLM-L6-v2", |
| "name": "all-MiniLM-L6-v2 (Dense Span+Null)", |
| "author": "sentence-transformers", |
| "system_type": "Open Weights \u00b7 Bi-Encoder", |
| "open_weights": true, |
| "params_m": 22.7, |
| "footprint_mb": 86.6, |
| "base_model": "sentence-transformers/all-MiniLM-L6-v2", |
| "runtime": "PyTorch MPS/CPU", |
| "input_rate_per_1m_usd": 0.007, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 35946, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0005, |
| "axes": { |
| "intelligence": 10.0, |
| "intelligence_ci95": [ |
| 7.7, |
| 12.3 |
| ], |
| "calibration": 70.9, |
| "speed": 67.1, |
| "cost": 88.8 |
| }, |
| "capability_score": 40.5, |
| "composite_score": 28.5, |
| "open_set_score": 11.2, |
| "sealed_set_score": 7.6, |
| "calibration_details": { |
| "null_auroc": 0.56307, |
| "null_ece": 0.30616, |
| "null_brier": 0.26323 |
| }, |
| "latency": { |
| "mean_ms": 5.0175, |
| "p50_ms": 4.9879, |
| "p95_ms": 5.187, |
| "p99_ms": 5.2385 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 8.3, |
| "n": 181 |
| }, |
| "math": { |
| "score": 0.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 28.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 44.4, |
| "n": 27 |
| }, |
| "score": { |
| "score": 0.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 0.5, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 25.0, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 15.3, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 16.4, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 0.9, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 23.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 0.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 19.1, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 4.9, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 0.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 1.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 10.4, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 18.8, |
| "n": 8 |
| } |
| }, |
| "official_rank": 19 |
| } |
| ], |
| "tiers": { |
| "tier_1a_pytorch_fp32": { |
| "id": "nullspan-fp32", |
| "name": "nullspan-345M (PyTorch FP32)", |
| "author": "gnumanth", |
| "system_type": "System One Encoder (5-Head, 512-ctx)", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 1317.2, |
| "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", |
| "runtime": "PyTorch MPS/CUDA/CPU FP32", |
| "input_rate_per_1m_usd": 0.012, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0017, |
| "axes": { |
| "intelligence": 98.0, |
| "intelligence_ci95": [ |
| 96.8, |
| 99.2 |
| ], |
| "calibration": 98.5, |
| "speed": 63.2, |
| "cost": 76.8 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 81.3, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.9, |
| "calibration_details": { |
| "null_auroc": 0.99327, |
| "null_ece": 0.01419, |
| "null_brier": 0.01532 |
| }, |
| "latency": { |
| "mean_ms": 16.9035, |
| "p50_ms": 10.2968, |
| "p95_ms": 33.0811, |
| "p99_ms": 37.3921 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 86.4, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 88.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 83.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 3 |
| }, |
| "tier_1b_onnx_int8": { |
| "id": "nullspan-onnx-int8", |
| "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", |
| "author": "gnumanth", |
| "system_type": "Quantized ONNX WASM/CPU (512-ctx)", |
| "open_weights": true, |
| "params_m": 345.3, |
| "footprint_mb": 331.1, |
| "base_model": "gnumanth/nullspan (Int8 Dynamic)", |
| "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", |
| "input_rate_per_1m_usd": 0.006, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0008, |
| "axes": { |
| "intelligence": 97.8, |
| "intelligence_ci95": [ |
| 96.6, |
| 99.0 |
| ], |
| "calibration": 98.6, |
| "speed": 52.7, |
| "cost": 84.2 |
| }, |
| "capability_score": 98.2, |
| "composite_score": 78.1, |
| "open_set_score": 100.0, |
| "sealed_set_score": 93.3, |
| "calibration_details": { |
| "null_auroc": 0.99511, |
| "null_ece": 0.01267, |
| "null_brier": 0.01523 |
| }, |
| "latency": { |
| "mean_ms": 108.0495, |
| "p50_ms": 75.9329, |
| "p95_ms": 207.004, |
| "p99_ms": 369.9251 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 96.3, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 84.1, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 96.5, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 86.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 81.7, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 4 |
| }, |
| "tier_2_zero_dep_engine": { |
| "id": "nullspan-zerodep-fast", |
| "name": "nullspan-ZeroDep (Fast Engine)", |
| "author": "gnumanth", |
| "system_type": "Zero-Dep Algorithmic Engine", |
| "open_weights": true, |
| "params_m": 0.0, |
| "footprint_mb": 0.0, |
| "base_model": "Deterministic Clause + Lexical Pointer", |
| "runtime": "Pure Python / Node / Browser JS", |
| "input_rate_per_1m_usd": 0.0008, |
| "output_rate_per_1m_usd": 0.0, |
| "n_observations": 500, |
| "input_tokens_total": 69943, |
| "output_tokens_total": 0, |
| "usd_per_1k_decisions": 0.0001, |
| "axes": { |
| "intelligence": 90.8, |
| "intelligence_ci95": [ |
| 88.2, |
| 93.4 |
| ], |
| "calibration": 95.0, |
| "speed": 93.5, |
| "cost": 99.8 |
| }, |
| "capability_score": 92.9, |
| "composite_score": 94.7, |
| "open_set_score": 100.0, |
| "sealed_set_score": 72.0, |
| "calibration_details": { |
| "null_auroc": 0.98239, |
| "null_ece": 0.04866, |
| "null_brier": 0.05142 |
| }, |
| "latency": { |
| "mean_ms": 0.1345, |
| "p50_ms": 0.0342, |
| "p95_ms": 0.6895, |
| "p99_ms": 0.8689 |
| }, |
| "by_request_type": { |
| "span_open": { |
| "score": 100.0, |
| "n": 181 |
| }, |
| "math": { |
| "score": 100.0, |
| "n": 51 |
| }, |
| "choice": { |
| "score": 100.0, |
| "n": 41 |
| }, |
| "noul_open": { |
| "score": 100.0, |
| "n": 27 |
| }, |
| "score": { |
| "score": 100.0, |
| "n": 48 |
| }, |
| "span_sealed": { |
| "score": 78.7, |
| "n": 108 |
| }, |
| "noul_sealed": { |
| "score": 47.7, |
| "n": 44 |
| } |
| }, |
| "by_tier_open": { |
| "Easy": { |
| "score": 100.0, |
| "n": 176 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "Standard": { |
| "score": 100.0, |
| "n": 64 |
| } |
| }, |
| "by_tier_sealed": { |
| "Standard": { |
| "score": 79.6, |
| "n": 113 |
| }, |
| "Hard": { |
| "score": 54.0, |
| "n": 50 |
| }, |
| "Judge": { |
| "score": 100.0, |
| "n": 1 |
| } |
| }, |
| "by_domain": { |
| "incident_report": { |
| "score": 100.0, |
| "n": 160 |
| }, |
| "compute_ledger": { |
| "score": 100.0, |
| "n": 144 |
| }, |
| "cluster_throughput_comps": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "edge_traffic_forecast": { |
| "score": 100.0, |
| "n": 16 |
| }, |
| "squad_v2_open_qa": { |
| "score": 23.3, |
| "n": 60 |
| }, |
| "contract_extraction": { |
| "score": 100.0, |
| "n": 96 |
| }, |
| "adversarial_stress": { |
| "score": 100.0, |
| "n": 8 |
| } |
| }, |
| "official_rank": 2 |
| } |
| } |
| } |
