{ "benchmark_name": "NullspanBench (JevBench-Compatible Decision & Extraction Leaderboard)", "benchmark_version": "3.2.0-jevbench", "draw_id": "S+P (draw v3.2-nullspan-20261030)", "timestamp": "2026-10-30T13:11:15Z", "n_total_observations": 500, "n_open_observations": 336, "n_sealed_observations": 164, "environment": { "os": "Linux 6.8.0-1070-gcp (x86_64)", "python": "3.10.12", "pytorch": "2.9.1+cu129", "onnxruntime": "1.23.2", "accelerator": "NVIDIA L4 (GCP CUDA Eval)" }, "systems": [ { "id": "nullspan-hybrid-router", "name": "nullspan-Hybrid (Fast + ONNX Int8)", "author": "gnumanth", "system_type": "Hybrid System One Router", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "Zero-Dep Router + gnumanth/nullspan Int8", "runtime": "Tiered JS/Python + ONNX Runtime", "input_rate_per_1m_usd": 0.002, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0003, "axes": { "intelligence": 97.8, "intelligence_ci95": [ 96.6, 99.0 ], "calibration": 98.6, "speed": 92.7, "cost": 93.8 }, "capability_score": 98.2, "composite_score": 95.7, "open_set_score": 100.0, "sealed_set_score": 93.3, "calibration_details": { "null_auroc": 0.99629, "null_ece": 0.01392, "null_brier": 0.01516 }, "latency": { "mean_ms": 16.8098, "p50_ms": 0.0397, "p95_ms": 126.6229, "p99_ms": 209.086 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 84.1, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 86.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 81.7, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 1 }, { "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "author": "gnumanth", "system_type": "Zero-Dep Algorithmic Engine", "open_weights": true, "params_m": 0.0, "footprint_mb": 0.0, "base_model": "Deterministic Clause + Lexical Pointer", "runtime": "Pure Python / Node / Browser JS", "input_rate_per_1m_usd": 0.0008, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0001, "axes": { "intelligence": 90.8, "intelligence_ci95": [ 88.2, 93.4 ], "calibration": 95.0, "speed": 93.5, "cost": 99.8 }, "capability_score": 92.9, "composite_score": 94.7, "open_set_score": 100.0, "sealed_set_score": 72.0, "calibration_details": { "null_auroc": 0.98239, "null_ece": 0.04866, "null_brier": 0.05142 }, "latency": { "mean_ms": 0.1345, "p50_ms": 0.0342, "p95_ms": 0.6895, "p99_ms": 0.8689 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 78.7, "n": 108 }, "noul_sealed": { "score": 47.7, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 79.6, "n": 113 }, "Hard": { "score": 54.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 23.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 2 }, { "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "author": "gnumanth", "system_type": "System One Encoder (5-Head, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1317.2, "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", "runtime": "PyTorch MPS/CUDA/CPU FP32", "input_rate_per_1m_usd": 0.012, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0017, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.5, "speed": 63.2, "cost": 76.8 }, "capability_score": 98.2, "composite_score": 81.3, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.99327, "null_ece": 0.01419, "null_brier": 0.01532 }, "latency": { "mean_ms": 16.9035, "p50_ms": 10.2968, "p95_ms": 33.0811, "p99_ms": 37.3921 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 3 }, { "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "author": "gnumanth", "system_type": "Quantized ONNX WASM/CPU (512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "gnumanth/nullspan (Int8 Dynamic)", "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", "input_rate_per_1m_usd": 0.006, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0008, "axes": { "intelligence": 97.8, "intelligence_ci95": [ 96.6, 99.0 ], "calibration": 98.6, "speed": 52.7, "cost": 84.2 }, "capability_score": 98.2, "composite_score": 78.1, "open_set_score": 100.0, "sealed_set_score": 93.3, "calibration_details": { "null_auroc": 0.99511, "null_ece": 0.01267, "null_brier": 0.01523 }, "latency": { "mean_ms": 108.0495, "p50_ms": 75.9329, "p95_ms": 207.004, "p99_ms": 369.9251 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 84.1, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 86.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 81.7, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 4 }, { "id": "h2o-lightning-4b", "name": "H2O-Lightning-4B v1.1 (JevBench #1 Comp)", "author": "H2O.ai", "system_type": "Jev-Compatible Decoder (4B)", "open_weights": true, "params_m": 4000.0, "footprint_mb": 8100.0, "base_model": "Qwen/Qwen3.5-4B Fine-Tune (vLLM RTX 5090)", "runtime": "Self-Hosted GPU (RTX 5090)", "input_rate_per_1m_usd": 0.027, "output_rate_per_1m_usd": 0.0, "n_observations": 1500, "input_tokens_total": 1152000, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0211, "axes": { "intelligence": 60.0, "intelligence_ci95": [ 57.8, 62.3 ], "calibration": 90.0, "speed": 92.6, "cost": 60.3 }, "capability_score": 75.0, "composite_score": 72.5, "open_set_score": 64.1, "sealed_set_score": 59.0, "calibration_details": { "null_auroc": 0.908, "null_ece": 0.0502, "null_brier": 0.0645 }, "latency": { "mean_ms": 34.1, "p50_ms": 29.285, "p95_ms": 58.4, "p99_ms": 74.0 }, "by_request_type": { "span_open": { "score": 61.5, "n": 300 }, "span_sealed": { "score": 58.2, "n": 300 }, "noul_open": { "score": 60.9, "n": 150 }, "noul_sealed": { "score": 68.1, "n": 150 }, "choice": { "score": 75.2, "n": 300 }, "math": { "score": 57.6, "n": 150 }, "score": { "score": 47.0, "n": 150 } }, "by_tier_open": { "Easy": { "score": 88.2, "n": 100 }, "Standard": { "score": 57.6, "n": 100 }, "Judge": { "score": 61.0, "n": 100 } }, "by_tier_sealed": { "Standard": { "score": 62.6, "n": 600 }, "Hard": { "score": 54.3, "n": 600 } }, "by_domain": { "incident_report": { "score": 69.4, "n": 300 }, "compute_ledger": { "score": 59.2, "n": 300 }, "cluster_throughput_comps": { "score": 47.0, "n": 150 }, "edge_traffic_forecast": { "score": 44.3, "n": 150 }, "squad_v2_open_qa": { "score": 62.6, "n": 300 }, "contract_extraction": { "score": 58.4, "n": 300 }, "adversarial_stress": { "score": 54.3, "n": 300 } }, "official_rank": 5 }, { "id": "cloudflare-clairic-clef-live", "name": "@cf/cloudflare/clairic + clef (Live Workers AI)", "author": "Cloudflare", "system_type": "System One Split API (Clairic + Clef 27B)", "open_weights": false, "params_m": 27000.0, "footprint_mb": null, "base_model": "@cf/cloudflare/clairic + @cf/cloudflare/clef (180 RPCs / 246 Passes)", "runtime": "Cloudflare Workers AI Edge RPC (p50 315.3 ms/doc)", "input_rate_per_1m_usd": 0.025, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 81206, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0041, "axes": { "intelligence": 91.6, "intelligence_ci95": [ 89.2, 93.9 ], "calibration": 94.9, "speed": 51.7, "cost": 68.2 }, "capability_score": 93.2, "composite_score": 72.1, "open_set_score": 89.7, "sealed_set_score": 95.4, "calibration_details": { "null_auroc": 0.98959, "null_ece": 0.05235, "null_brier": 0.04728 }, "latency": { "mean_ms": 128.789, "p50_ms": 91.323, "p95_ms": 314.023, "p99_ms": 403.1387 }, "by_request_type": { "span_open": { "score": 83.8, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 89.0, "n": 48 }, "span_sealed": { "score": 95.8, "n": 108 }, "noul_sealed": { "score": 93.2, "n": 44 } }, "by_tier_open": { "Easy": { "score": 85.8, "n": 176 }, "Judge": { "score": 94.5, "n": 96 }, "Standard": { "score": 93.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.9, "n": 113 }, "Hard": { "score": 92.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 94.4, "n": 160 }, "compute_ledger": { "score": 82.6, "n": 144 }, "cluster_throughput_comps": { "score": 96.2, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 92.5, "n": 60 }, "contract_extraction": { "score": 97.9, "n": 96 }, "adversarial_stress": { "score": 87.5, "n": 8 } }, "official_rank": 6 }, { "id": "basal-omni-1.0-ultra", "name": "basal-omni-1.0-ultra (JevBench #1 Cap)", "author": "Remek Kinas", "system_type": "Jev-Compatible Decoder (31B)", "open_weights": true, "params_m": 31000.0, "footprint_mb": 62000.0, "base_model": "Gemma-4-31B-it + Merged LoRA (Typed Readout)", "runtime": "Self-Hosted GPU (p50 102 ms)", "input_rate_per_1m_usd": 0.058, "output_rate_per_1m_usd": 0.0, "n_observations": 1500, "input_tokens_total": 1152000, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0447, "axes": { "intelligence": 74.1, "intelligence_ci95": [ 71.8, 76.3 ], "calibration": 91.0, "speed": 87.3, "cost": 50.5 }, "capability_score": 82.5, "composite_score": 71.8, "open_set_score": 76.7, "sealed_set_score": 73.1, "calibration_details": { "null_auroc": 0.924, "null_ece": 0.0449, "null_brier": 0.055 }, "latency": { "mean_ms": 114.2, "p50_ms": 102.0, "p95_ms": 168.0, "p99_ms": 210.0 }, "by_request_type": { "span_open": { "score": 75.4, "n": 300 }, "span_sealed": { "score": 73.8, "n": 300 }, "noul_open": { "score": 68.4, "n": 150 }, "noul_sealed": { "score": 72.5, "n": 150 }, "choice": { "score": 82.7, "n": 300 }, "math": { "score": 69.1, "n": 150 }, "score": { "score": 69.1, "n": 150 } }, "by_tier_open": { "Easy": { "score": 96.5, "n": 100 }, "Standard": { "score": 69.1, "n": 100 }, "Judge": { "score": 65.2, "n": 100 } }, "by_tier_sealed": { "Standard": { "score": 78.8, "n": 600 }, "Hard": { "score": 74.0, "n": 600 } }, "by_domain": { "incident_report": { "score": 81.2, "n": 300 }, "compute_ledger": { "score": 71.5, "n": 300 }, "cluster_throughput_comps": { "score": 69.1, "n": 300 }, "edge_traffic_forecast": { "score": 66.9, "n": 150 }, "squad_v2_open_qa": { "score": 76.9, "n": 300 }, "contract_extraction": { "score": 72.8, "n": 300 }, "adversarial_stress": { "score": 74.0, "n": 300 } }, "official_rank": 7 }, { "id": "jev-1.13.0-ref", "name": "Jev 1.13.0 (TypeSafe AI \u00b7 JevBench Ref)", "author": "TypeSafe AI", "system_type": "Jev Reference \u00b7 Hosted API", "open_weights": false, "params_m": null, "footprint_mb": null, "base_model": "TypeSafe Jev 1.13.0 ($0.042/1M input)", "runtime": "TypeSafe Hosted API (JevBench v1.7)", "input_rate_per_1m_usd": 0.042, "output_rate_per_1m_usd": 0.0, "n_observations": 1500, "input_tokens_total": 1152000, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0323, "axes": { "intelligence": 63.6, "intelligence_ci95": [ 61.2, 66.0 ], "calibration": 90.6, "speed": 91.5, "cost": 54.7 }, "capability_score": 77.1, "composite_score": 71.5, "open_set_score": 68.9, "sealed_set_score": 61.6, "calibration_details": { "null_auroc": 0.912, "null_ece": 0.0468, "null_brier": 0.0612 }, "latency": { "mean_ms": 251.4, "p50_ms": 239.102, "p95_ms": 412.0, "p99_ms": 530.0 }, "by_request_type": { "span_open": { "score": 66.2, "n": 300 }, "span_sealed": { "score": 61.4, "n": 300 }, "noul_open": { "score": 54.5, "n": 150 }, "noul_sealed": { "score": 46.1, "n": 150 }, "choice": { "score": 79.0, "n": 300 }, "math": { "score": 63.2, "n": 150 }, "score": { "score": 63.3, "n": 150 } }, "by_tier_open": { "Easy": { "score": 93.9, "n": 100 }, "Standard": { "score": 63.2, "n": 100 }, "Judge": { "score": 67.7, "n": 100 } }, "by_tier_sealed": { "Standard": { "score": 68.7, "n": 600 }, "Hard": { "score": 58.9, "n": 600 } }, "by_domain": { "incident_report": { "score": 74.2, "n": 300 }, "compute_ledger": { "score": 66.8, "n": 300 }, "cluster_throughput_comps": { "score": 63.3, "n": 150 }, "edge_traffic_forecast": { "score": 63.0, "n": 150 }, "squad_v2_open_qa": { "score": 61.6, "n": 300 }, "contract_extraction": { "score": 62.4, "n": 300 }, "adversarial_stress": { "score": 58.9, "n": 300 } }, "official_rank": 8 }, { "id": "nullspan-onnx-fp32", "name": "nullspan-345M (ONNX FP32)", "author": "gnumanth", "system_type": "System One Encoder (ONNX, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1317.6, "base_model": "gnumanth/nullspan (ONNX Opset 17)", "runtime": "ONNX Runtime FP32", "input_rate_per_1m_usd": 0.011, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0015, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.5, "speed": 43.5, "cost": 78.0 }, "capability_score": 98.2, "composite_score": 71.2, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.99327, "null_ece": 0.01419, "null_brier": 0.01532 }, "latency": { "mean_ms": 491.2899, "p50_ms": 424.9432, "p95_ms": 1074.4535, "p99_ms": 1256.8557 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 9 }, { "id": "deberta-v3-base-squad2", "name": "deberta-v3-base-squad2 (184M SQuAD2 QA)", "author": "deepset", "system_type": "Open Weights \u00b7 Extractive QA Cross-Encoder", "open_weights": true, "params_m": 183.8, "footprint_mb": 701.3, "base_model": "deepset/deberta-v3-base-squad2", "runtime": "PyTorch MPS/CPU (512-ctx)", "input_rate_per_1m_usd": 0.032, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 57964, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0037, "axes": { "intelligence": 48.1, "intelligence_ci95": [ 43.9, 52.4 ], "calibration": 63.4, "speed": 65.1, "cost": 69.2 }, "capability_score": 55.8, "composite_score": 60.3, "open_set_score": 48.6, "sealed_set_score": 47.3, "calibration_details": { "null_auroc": 0.82087, "null_ece": 0.39493, "null_brier": 0.31346 }, "latency": { "mean_ms": 9.6505, "p50_ms": 7.2341, "p95_ms": 24.8532, "p99_ms": 26.2559 }, "by_request_type": { "span_open": { "score": 56.4, "n": 181 }, "math": { "score": 4.9, "n": 51 }, "choice": { "score": 65.9, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 15.1, "n": 48 }, "span_sealed": { "score": 35.2, "n": 108 }, "noul_sealed": { "score": 84.1, "n": 44 } }, "by_tier_open": { "Easy": { "score": 64.2, "n": 176 }, "Judge": { "score": 9.6, "n": 96 }, "Standard": { "score": 64.1, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 33.6, "n": 113 }, "Hard": { "score": 78.0, "n": 50 }, "Judge": { "score": 50.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 50.0, "n": 160 }, "compute_ledger": { "score": 57.6, "n": 144 }, "cluster_throughput_comps": { "score": 1.3, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 80.0, "n": 60 }, "contract_extraction": { "score": 27.6, "n": 96 }, "adversarial_stress": { "score": 37.5, "n": 8 } }, "official_rank": 10 }, { "id": "gpt-5.4-nano-json", "name": "gpt-5.4-nano (JSON Decoder \u00b7 API)", "author": "OpenAI (Ref)", "system_type": "Closed API \u00b7 LLM Decoder", "open_weights": false, "params_m": null, "footprint_mb": null, "base_model": "Autoregressive JSON Schema Decoder", "runtime": "Hosted Cloud API ( ~38 output tokens/field )", "input_rate_per_1m_usd": 0.15, "output_rate_per_1m_usd": 0.6, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 19000, "usd_per_1k_decisions": 0.0438, "axes": { "intelligence": 95.2, "intelligence_ci95": [ 93.4, 97.0 ], "calibration": 89.0, "speed": 43.6, "cost": 45.1 }, "capability_score": 92.1, "composite_score": 59.8, "open_set_score": 99.7, "sealed_set_score": 86.0, "calibration_details": { "null_auroc": 0.95791, "null_ece": 0.15112, "null_brier": 0.03403 }, "latency": { "mean_ms": 420.4635, "p50_ms": 417.9908, "p95_ms": 515.593, "p99_ms": 524.2688 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 96.3, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 91.7, "n": 108 }, "noul_sealed": { "score": 68.2, "n": 44 } }, "by_tier_open": { "Easy": { "score": 99.4, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 92.0, "n": 113 }, "Hard": { "score": 72.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 99.4, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 76.7, "n": 60 }, "contract_extraction": { "score": 91.7, "n": 96 }, "adversarial_stress": { "score": 87.5, "n": 8 } }, "official_rank": 11 }, { "id": "claude-haiku-4.5-json", "name": "claude-haiku-4.5 (JSON Decoder \u00b7 API)", "author": "Anthropic (Ref)", "system_type": "Closed API \u00b7 LLM Decoder", "open_weights": false, "params_m": null, "footprint_mb": null, "base_model": "Autoregressive JSON Schema Decoder", "runtime": "Hosted Cloud API ( ~38 output tokens/field )", "input_rate_per_1m_usd": 0.8, "output_rate_per_1m_usd": 4.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 19000, "usd_per_1k_decisions": 0.2639, "axes": { "intelligence": 95.4, "intelligence_ci95": [ 93.4, 97.2 ], "calibration": 88.7, "speed": 41.8, "cost": 27.5 }, "capability_score": 92.1, "composite_score": 48.8, "open_set_score": 99.4, "sealed_set_score": 87.2, "calibration_details": { "null_auroc": 0.98592, "null_ece": 0.1566, "null_brier": 0.03222 }, "latency": { "mean_ms": 585.679, "p50_ms": 586.3918, "p95_ms": 717.8024, "p99_ms": 727.6919 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 92.6, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 92.6, "n": 108 }, "noul_sealed": { "score": 70.5, "n": 44 } }, "by_tier_open": { "Easy": { "score": 98.9, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 92.9, "n": 113 }, "Hard": { "score": 74.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 99.4, "n": 160 }, "compute_ledger": { "score": 99.3, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 78.3, "n": 60 }, "contract_extraction": { "score": 93.8, "n": 96 }, "adversarial_stress": { "score": 75.0, "n": 8 } }, "official_rank": 12 }, { "id": "bge-small-en-v1.5", "name": "bge-small-en-v1.5 (Dense Span+Null)", "author": "BAAI", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 33.4, "footprint_mb": 127.3, "base_model": "BAAI/bge-small-en-v1.5", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.01, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0007, "axes": { "intelligence": 14.7, "intelligence_ci95": [ 12.0, 17.5 ], "calibration": 89.2, "speed": 64.2, "cost": 85.5 }, "capability_score": 52.0, "composite_score": 37.6, "open_set_score": 20.3, "sealed_set_score": 3.4, "calibration_details": { "null_auroc": 0.68694, "null_ece": 0.10548, "null_brier": 0.11334 }, "latency": { "mean_ms": 8.6668, "p50_ms": 8.6253, "p95_ms": 8.9805, "p99_ms": 9.3022 }, "by_request_type": { "span_open": { "score": 12.2, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 59.8, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 24.4, "n": 48 }, "span_sealed": { "score": 2.3, "n": 108 }, "noul_sealed": { "score": 2.3, "n": 44 } }, "by_tier_open": { "Easy": { "score": 19.3, "n": 176 }, "Judge": { "score": 12.2, "n": 96 }, "Standard": { "score": 35.2, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 2.7, "n": 113 }, "Hard": { "score": 5.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 29.9, "n": 160 }, "compute_ledger": { "score": 9.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 20.1, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 3.1, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 13 }, { "id": "distilbert-base-control", "name": "distilbert-base-control (Raw Control)", "author": "distilbert (control)", "system_type": "Base Model Control (Untuned)", "open_weights": true, "params_m": 66.4, "footprint_mb": 253.2, "base_model": "distilbert/distilbert-base-uncased", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.015, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0011, "axes": { "intelligence": 14.2, "intelligence_ci95": [ 11.6, 17.0 ], "calibration": 91.5, "speed": 67.7, "cost": 81.1 }, "capability_score": 52.9, "composite_score": 36.9, "open_set_score": 20.2, "sealed_set_score": 1.8, "calibration_details": { "null_auroc": 0.60728, "null_ece": 0.07034, "null_brier": 0.11186 }, "latency": { "mean_ms": 4.552, "p50_ms": 4.4787, "p95_ms": 5.4739, "p99_ms": 5.5359 }, "by_request_type": { "span_open": { "score": 16.0, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 59.8, "n": 41 }, "noul_open": { "score": 0.0, "n": 27 }, "score": { "score": 34.2, "n": 48 }, "span_sealed": { "score": 0.9, "n": 108 }, "noul_sealed": { "score": 0.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 16.5, "n": 176 }, "Judge": { "score": 17.1, "n": 96 }, "Standard": { "score": 35.2, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 1.8, "n": 113 }, "Hard": { "score": 2.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 24.2, "n": 160 }, "compute_ledger": { "score": 11.1, "n": 144 }, "cluster_throughput_comps": { "score": 27.4, "n": 16 }, "edge_traffic_forecast": { "score": 55.1, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 1.0, "n": 96 }, "adversarial_stress": { "score": 12.5, "n": 8 } }, "official_rank": 14 }, { "id": "all-MiniLM-L12-v2", "name": "all-MiniLM-L12-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 33.4, "footprint_mb": 127.3, "base_model": "sentence-transformers/all-MiniLM-L12-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.01, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0007, "axes": { "intelligence": 14.4, "intelligence_ci95": [ 11.7, 17.5 ], "calibration": 73.5, "speed": 64.2, "cost": 85.5 }, "capability_score": 44.0, "composite_score": 36.3, "open_set_score": 17.2, "sealed_set_score": 8.8, "calibration_details": { "null_auroc": 0.58938, "null_ece": 0.28318, "null_brier": 0.23193 }, "latency": { "mean_ms": 8.6507, "p50_ms": 8.5815, "p95_ms": 8.9115, "p99_ms": 9.0401 }, "by_request_type": { "span_open": { "score": 9.4, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 6.7, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 29.5, "n": 44 } }, "by_tier_open": { "Easy": { "score": 25.0, "n": 176 }, "Judge": { "score": 3.3, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 27.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 16.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 11.5, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 15 }, { "id": "all-mpnet-base-v2", "name": "all-mpnet-base-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 109.5, "footprint_mb": 417.7, "base_model": "sentence-transformers/all-mpnet-base-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.022, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0016, "axes": { "intelligence": 13.6, "intelligence_ci95": [ 10.9, 16.7 ], "calibration": 71.0, "speed": 63.3, "cost": 77.4 }, "capability_score": 42.3, "composite_score": 34.4, "open_set_score": 16.2, "sealed_set_score": 8.2, "calibration_details": { "null_auroc": 0.58731, "null_ece": 0.30424, "null_brier": 0.26407 }, "latency": { "mean_ms": 10.2958, "p50_ms": 10.2572, "p95_ms": 10.5562, "p99_ms": 12.7047 }, "by_request_type": { "span_open": { "score": 9.4, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 0.0, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 27.3, "n": 44 } }, "by_tier_open": { "Easy": { "score": 25.0, "n": 176 }, "Judge": { "score": 0.0, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 25.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 16.7, "n": 144 }, "cluster_throughput_comps": { "score": 0.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 11.5, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 16 }, { "id": "paraphrase-albert-small-v2", "name": "paraphrase-albert-small-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 11.7, "footprint_mb": 44.6, "base_model": "sentence-transformers/paraphrase-albert-small-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.005, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0004, "axes": { "intelligence": 12.5, "intelligence_ci95": [ 10.0, 15.2 ], "calibration": 76.6, "speed": 66.4, "cost": 91.0 }, "capability_score": 44.5, "composite_score": 33.6, "open_set_score": 15.3, "sealed_set_score": 6.7, "calibration_details": { "null_auroc": 0.58482, "null_ece": 0.24547, "null_brier": 0.21391 }, "latency": { "mean_ms": 5.7054, "p50_ms": 5.7004, "p95_ms": 5.8183, "p99_ms": 5.8798 }, "by_request_type": { "span_open": { "score": 11.0, "n": 181 }, "math": { "score": 1.0, "n": 51 }, "choice": { "score": 35.4, "n": 41 }, "noul_open": { "score": 51.9, "n": 27 }, "score": { "score": 8.4, "n": 48 }, "span_sealed": { "score": 2.3, "n": 108 }, "noul_sealed": { "score": 15.9, "n": 44 } }, "by_tier_open": { "Easy": { "score": 19.3, "n": 176 }, "Judge": { "score": 4.2, "n": 96 }, "Standard": { "score": 21.1, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 2.7, "n": 113 }, "Hard": { "score": 15.0, "n": 50 }, "Judge": { "score": 50.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 21.1, "n": 160 }, "compute_ledger": { "score": 9.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 3.4, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 9.9, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 17 }, { "id": "paraphrase-MiniLM-L3-v2", "name": "paraphrase-MiniLM-L3-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 17.4, "footprint_mb": 66.3, "base_model": "sentence-transformers/paraphrase-MiniLM-L3-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.005, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0004, "axes": { "intelligence": 11.1, "intelligence_ci95": [ 8.7, 13.8 ], "calibration": 72.3, "speed": 69.6, "cost": 91.0 }, "capability_score": 41.7, "composite_score": 30.9, "open_set_score": 12.9, "sealed_set_score": 7.6, "calibration_details": { "null_auroc": 0.58932, "null_ece": 0.29025, "null_brier": 0.25176 }, "latency": { "mean_ms": 3.1323, "p50_ms": 3.1142, "p95_ms": 3.2437, "p99_ms": 3.2779 }, "by_request_type": { "span_open": { "score": 9.7, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 6.7, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 25.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 16.8, "n": 176 }, "Judge": { "score": 3.3, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 23.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 17.5, "n": 160 }, "compute_ledger": { "score": 8.3, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 10.4, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 18 }, { "id": "all-MiniLM-L6-v2", "name": "all-MiniLM-L6-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 22.7, "footprint_mb": 86.6, "base_model": "sentence-transformers/all-MiniLM-L6-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.007, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0005, "axes": { "intelligence": 10.0, "intelligence_ci95": [ 7.7, 12.3 ], "calibration": 70.9, "speed": 67.1, "cost": 88.8 }, "capability_score": 40.5, "composite_score": 28.5, "open_set_score": 11.2, "sealed_set_score": 7.6, "calibration_details": { "null_auroc": 0.56307, "null_ece": 0.30616, "null_brier": 0.26323 }, "latency": { "mean_ms": 5.0175, "p50_ms": 4.9879, "p95_ms": 5.187, "p99_ms": 5.2385 }, "by_request_type": { "span_open": { "score": 8.3, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 0.0, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 25.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 15.3, "n": 176 }, "Judge": { "score": 0.0, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 23.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 4.9, "n": 144 }, "cluster_throughput_comps": { "score": 0.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 10.4, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 19 } ], "tiers": { "tier_1a_pytorch_fp32": { "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "author": "gnumanth", "system_type": "System One Encoder (5-Head, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1317.2, "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", "runtime": "PyTorch MPS/CUDA/CPU FP32", "input_rate_per_1m_usd": 0.012, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0017, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.5, "speed": 63.2, "cost": 76.8 }, "capability_score": 98.2, "composite_score": 81.3, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.99327, "null_ece": 0.01419, "null_brier": 0.01532 }, "latency": { "mean_ms": 16.9035, "p50_ms": 10.2968, "p95_ms": 33.0811, "p99_ms": 37.3921 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 3 }, "tier_1b_onnx_int8": { "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "author": "gnumanth", "system_type": "Quantized ONNX WASM/CPU (512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "gnumanth/nullspan (Int8 Dynamic)", "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", "input_rate_per_1m_usd": 0.006, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0008, "axes": { "intelligence": 97.8, "intelligence_ci95": [ 96.6, 99.0 ], "calibration": 98.6, "speed": 52.7, "cost": 84.2 }, "capability_score": 98.2, "composite_score": 78.1, "open_set_score": 100.0, "sealed_set_score": 93.3, "calibration_details": { "null_auroc": 0.99511, "null_ece": 0.01267, "null_brier": 0.01523 }, "latency": { "mean_ms": 108.0495, "p50_ms": 75.9329, "p95_ms": 207.004, "p99_ms": 369.9251 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 84.1, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 86.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 81.7, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 4 }, "tier_2_zero_dep_engine": { "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "author": "gnumanth", "system_type": "Zero-Dep Algorithmic Engine", "open_weights": true, "params_m": 0.0, "footprint_mb": 0.0, "base_model": "Deterministic Clause + Lexical Pointer", "runtime": "Pure Python / Node / Browser JS", "input_rate_per_1m_usd": 0.0008, "output_rate_per_1m_usd": 0.0, "n_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0001, "axes": { "intelligence": 90.8, "intelligence_ci95": [ 88.2, 93.4 ], "calibration": 95.0, "speed": 93.5, "cost": 99.8 }, "capability_score": 92.9, "composite_score": 94.7, "open_set_score": 100.0, "sealed_set_score": 72.0, "calibration_details": { "null_auroc": 0.98239, "null_ece": 0.04866, "null_brier": 0.05142 }, "latency": { "mean_ms": 0.1345, "p50_ms": 0.0342, "p95_ms": 0.6895, "p99_ms": 0.8689 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 78.7, "n": 108 }, "noul_sealed": { "score": 47.7, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 79.6, "n": 113 }, "Hard": { "score": 54.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 23.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 2 } } }