Release v1.2.0: 344.2M ELECTRA-Large + 330.1 MB Int8 ONNX, 19-system NullspanBench N=500, and contract preset
Browse files- README.md +2 -2
- app.py +21 -2
- bench-data.json +920 -633
- bench.html +78 -9
- engine.py +83 -23
- flow-diagram.svg +2 -2
- index.html +230 -89
- nullspan-engine.js +67 -9
- nullspan-onnx-browser.js +218 -110
- nullspan-vs-clairic.gif +2 -2
- nullspan-vs-clairic.mp4 +2 -2
- tokenizer.json +1 -1
README.md
CHANGED
|
@@ -15,9 +15,9 @@ short_description: System One structured extraction, p_null & quantiles
|
|
| 15 |
|
| 16 |
# `nullspan` — System One Structured Extraction, Calibrated Abstention (`p_null`) & Quantile Distribution Space
|
| 17 |
|
| 18 |
-
Interactive Hugging Face Space for **[`gnumanth/nullspan`](https://huggingface.co/gnumanth/nullspan)** (`
|
| 19 |
|
| 20 |
-
- **In-Browser Neural Execution**: Click **
|
| 21 |
- **Local Gradio Python App**: Also includes [`app.py`](./app.py) (`python app.py`) for PyTorch FP32 Gradio execution.
|
| 22 |
- **GitHub Repository**: [`hemanth/nullspan`](https://github.com/hemanth/nullspan)
|
| 23 |
- **Dataset**: [`gnumanth/nullspan-dataset`](https://huggingface.co/datasets/gnumanth/nullspan-dataset)
|
|
|
|
| 15 |
|
| 16 |
# `nullspan` — System One Structured Extraction, Calibrated Abstention (`p_null`) & Quantile Distribution Space
|
| 17 |
|
| 18 |
+
Interactive Hugging Face Space for **[`gnumanth/nullspan`](https://huggingface.co/gnumanth/nullspan)** (`344.2M` parameters, 512-token `ELECTRA-Large` SQuAD2 cross-encoder + 2-layer `SchemaAdapterBlock` + 10-way System One operation router + calibrated Pinball-loss quantile head).
|
| 19 |
|
| 20 |
+
- **In-Browser Neural Execution**: Click **Tier 1: In-Browser Neural ONNX (WASM SIMD · 330.1 MB Int8)** in the toolbar to stream and run [`gnumanth/nullspan/onnx/model_quantized.onnx`](https://huggingface.co/gnumanth/nullspan/blob/main/onnx/model_quantized.onnx) directly inside your browser via WebAssembly SIMD.
|
| 21 |
- **Local Gradio Python App**: Also includes [`app.py`](./app.py) (`python app.py`) for PyTorch FP32 Gradio execution.
|
| 22 |
- **GitHub Repository**: [`hemanth/nullspan`](https://github.com/hemanth/nullspan)
|
| 23 |
- **Dataset**: [`gnumanth/nullspan-dataset`](https://huggingface.co/datasets/gnumanth/nullspan-dataset)
|
app.py
CHANGED
|
@@ -160,6 +160,25 @@ PRESETS: dict[str, dict[str, Any]] = {
|
|
| 160 |
"unanswerable_trap": {"type": "text", "instructions": "What other countries besides France did the Normans give their name to?"},
|
| 161 |
},
|
| 162 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
}
|
| 164 |
|
| 165 |
|
|
@@ -236,7 +255,7 @@ def run_nullspan(engine_mode: str, state: str, schema_json: str) -> tuple[str, l
|
|
| 236 |
else:
|
| 237 |
ext = NullspanExtractor(fields_spec)
|
| 238 |
raw_out = ext.run(state)
|
| 239 |
-
fields_out = raw_out.get("fields", {})
|
| 240 |
payload = {
|
| 241 |
"engine": "Zero-Dep Algorithmic Fast Engine (nullspan)",
|
| 242 |
"latency_ms": round((time.perf_counter() - t0) * 1000.0, 3),
|
|
@@ -285,7 +304,7 @@ with gr.Blocks(title="nullspan — System One Structured Extraction & Quantiles"
|
|
| 285 |
gr.Markdown(
|
| 286 |
"""
|
| 287 |
# `nullspan` — System One Structured Extraction, Calibrated `p_null` & Quantile Distributions
|
| 288 |
-
Single-pass encoder extraction (`gnumanth/nullspan`,
|
| 289 |
- **Model Hub**: [`gnumanth/nullspan`](https://huggingface.co/gnumanth/nullspan) · **Dataset**: [`gnumanth/nullspan-dataset`](https://huggingface.co/datasets/gnumanth/nullspan-dataset) · **GitHub & Browser WebGPU/WASM Playground**: [`hemanth/nullspan`](https://hemanth.github.io/nullspan/)
|
| 290 |
"""
|
| 291 |
)
|
|
|
|
| 160 |
"unanswerable_trap": {"type": "text", "instructions": "What other countries besides France did the Normans give their name to?"},
|
| 161 |
},
|
| 162 |
},
|
| 163 |
+
"7. Enterprise MSA Contract (Multi-Numeric Disambiguation + >512-Token Windowing)": {
|
| 164 |
+
"state": (
|
| 165 |
+
"MASTER SUBSCRIPTION AGREEMENT #MSA-2026-8841\n"
|
| 166 |
+
"Effective Date: March 1, 2026 | Initial Term: 36 months | Auto-Renewal Notice Window: 60 days prior to expiration.\n"
|
| 167 |
+
"Section 4.1 Fees & Payment: Customer commits to 250 Enterprise Seats at $140.00 per seat/month, plus Platform Base Fee of $4,500.00/month. "
|
| 168 |
+
"Annual Contract Value (ACV): $474,000.00. Invoices are payable Net 45 days; late balances accrue interest at 1.5% per month.\n"
|
| 169 |
+
"Section 7.2 Limitation of Liability: General Liability Cap shall not exceed $948,000.00 (2x ACV); Super-Cap for Data Breach & IP Indemnity is $2,500,000.00. "
|
| 170 |
+
"Early Termination Fee: not applicable (waived under Addendum B)."
|
| 171 |
+
),
|
| 172 |
+
"schema": {
|
| 173 |
+
"initial_term_months": {"type": "number", "integer": True, "instructions": "Initial contract term length in months"},
|
| 174 |
+
"seat_unit_price": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Monthly price per Enterprise Seat"},
|
| 175 |
+
"annual_contract_value": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Annual Contract Value (ACV)"},
|
| 176 |
+
"payment_net_days": {"type": "number", "integer": True, "instructions": "Net payment terms in days"},
|
| 177 |
+
"general_liability_cap": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "General limitation of liability cap"},
|
| 178 |
+
"super_cap_indemnity": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Super-cap for data breach and IP indemnity"},
|
| 179 |
+
"early_termination_fee": {"type": "number", "decimals": 2, "unit": "USD", "instructions": "Early termination fee amount if applicable"},
|
| 180 |
+
},
|
| 181 |
+
},
|
| 182 |
}
|
| 183 |
|
| 184 |
|
|
|
|
| 255 |
else:
|
| 256 |
ext = NullspanExtractor(fields_spec)
|
| 257 |
raw_out = ext.run(state)
|
| 258 |
+
fields_out = raw_out.get("answers", raw_out.get("fields", {}))
|
| 259 |
payload = {
|
| 260 |
"engine": "Zero-Dep Algorithmic Fast Engine (nullspan)",
|
| 261 |
"latency_ms": round((time.perf_counter() - t0) * 1000.0, 3),
|
|
|
|
| 304 |
gr.Markdown(
|
| 305 |
"""
|
| 306 |
# `nullspan` — System One Structured Extraction, Calibrated `p_null` & Quantile Distributions
|
| 307 |
+
Single-pass encoder extraction (`gnumanth/nullspan`, 344.2M params, 512-token context + long-doc windowing) with character-offset `[start, end]` grounding, calibrated null abstention (`p_null`), 10-way System One operation routing (`BOOL_JUDGE`, `ENUM_CLASSIFY`, `ARITH_SUM`, `ARITH_MUL`), cross-field algebraic invariant verification, and Pinball-loss quantile distributions (`p10`, `median`, `p90`).
|
| 308 |
- **Model Hub**: [`gnumanth/nullspan`](https://huggingface.co/gnumanth/nullspan) · **Dataset**: [`gnumanth/nullspan-dataset`](https://huggingface.co/datasets/gnumanth/nullspan-dataset) · **GitHub & Browser WebGPU/WASM Playground**: [`hemanth/nullspan`](https://hemanth.github.io/nullspan/)
|
| 309 |
"""
|
| 310 |
)
|
bench-data.json
CHANGED
|
@@ -1,17 +1,17 @@
|
|
| 1 |
{
|
| 2 |
"benchmark_name": "NullspanBench (JevBench-Compatible Decision & Extraction Leaderboard)",
|
| 3 |
-
"benchmark_version": "3.
|
| 4 |
-
"draw_id": "S+P (draw v3.
|
| 5 |
-
"timestamp": "2026-10-
|
| 6 |
-
"n_total_observations":
|
| 7 |
"n_open_observations": 336,
|
| 8 |
-
"n_sealed_observations":
|
| 9 |
"environment": {
|
| 10 |
"os": "Darwin 25.5.0 (arm64)",
|
| 11 |
"python": "3.14.8",
|
| 12 |
"pytorch": "2.12.1",
|
| 13 |
"onnxruntime": "1.31.0",
|
| 14 |
-
"accelerator": "Apple Silicon MPS
|
| 15 |
},
|
| 16 |
"systems": [
|
| 17 |
{
|
|
@@ -20,40 +20,40 @@
|
|
| 20 |
"author": "gnumanth",
|
| 21 |
"system_type": "Hybrid System One Router",
|
| 22 |
"open_weights": true,
|
| 23 |
-
"params_m":
|
| 24 |
-
"footprint_mb":
|
| 25 |
"base_model": "Zero-Dep Router + gnumanth/nullspan Int8",
|
| 26 |
"runtime": "Tiered JS/Python + ONNX Runtime",
|
| 27 |
-
"input_rate_per_1m_usd": 0.
|
| 28 |
"output_rate_per_1m_usd": 0.0,
|
| 29 |
-
"n_observations":
|
| 30 |
-
"input_tokens_total":
|
| 31 |
"output_tokens_total": 0,
|
| 32 |
"usd_per_1k_decisions": 0.0003,
|
| 33 |
"axes": {
|
| 34 |
-
"intelligence":
|
| 35 |
"intelligence_ci95": [
|
| 36 |
-
96.
|
| 37 |
-
99.
|
| 38 |
],
|
| 39 |
-
"calibration":
|
| 40 |
-
"speed":
|
| 41 |
"cost": 93.8
|
| 42 |
},
|
| 43 |
-
"capability_score": 98.
|
| 44 |
-
"composite_score": 97.
|
| 45 |
"open_set_score": 100.0,
|
| 46 |
-
"sealed_set_score":
|
| 47 |
"calibration_details": {
|
| 48 |
-
"null_auroc": 0.
|
| 49 |
-
"null_ece": 0.
|
| 50 |
-
"null_brier": 0.
|
| 51 |
},
|
| 52 |
"latency": {
|
| 53 |
-
"mean_ms":
|
| 54 |
-
"p50_ms": 0.
|
| 55 |
-
"p95_ms":
|
| 56 |
-
"p99_ms":
|
| 57 |
},
|
| 58 |
"by_request_type": {
|
| 59 |
"span_open": {
|
|
@@ -62,11 +62,11 @@
|
|
| 62 |
},
|
| 63 |
"math": {
|
| 64 |
"score": 100.0,
|
| 65 |
-
"n":
|
| 66 |
},
|
| 67 |
"choice": {
|
| 68 |
"score": 100.0,
|
| 69 |
-
"n":
|
| 70 |
},
|
| 71 |
"noul_open": {
|
| 72 |
"score": 100.0,
|
|
@@ -77,12 +77,12 @@
|
|
| 77 |
"n": 48
|
| 78 |
},
|
| 79 |
"span_sealed": {
|
| 80 |
-
"score":
|
| 81 |
-
"n":
|
| 82 |
},
|
| 83 |
"noul_sealed": {
|
| 84 |
-
"score":
|
| 85 |
-
"n":
|
| 86 |
}
|
| 87 |
},
|
| 88 |
"by_tier_open": {
|
|
@@ -101,12 +101,16 @@
|
|
| 101 |
},
|
| 102 |
"by_tier_sealed": {
|
| 103 |
"Standard": {
|
| 104 |
-
"score":
|
| 105 |
-
"n":
|
| 106 |
},
|
| 107 |
"Hard": {
|
| 108 |
-
"score":
|
| 109 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
}
|
| 111 |
},
|
| 112 |
"by_domain": {
|
|
@@ -127,9 +131,13 @@
|
|
| 127 |
"n": 16
|
| 128 |
},
|
| 129 |
"squad_v2_open_qa": {
|
| 130 |
-
"score":
|
| 131 |
"n": 60
|
| 132 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
"adversarial_stress": {
|
| 134 |
"score": 100.0,
|
| 135 |
"n": 8
|
|
@@ -145,38 +153,38 @@
|
|
| 145 |
"open_weights": true,
|
| 146 |
"params_m": 0.0,
|
| 147 |
"footprint_mb": 0.0,
|
| 148 |
-
"base_model": "Deterministic
|
| 149 |
"runtime": "Pure Python / Node / Browser JS",
|
| 150 |
"input_rate_per_1m_usd": 0.0008,
|
| 151 |
"output_rate_per_1m_usd": 0.0,
|
| 152 |
-
"n_observations":
|
| 153 |
-
"input_tokens_total":
|
| 154 |
"output_tokens_total": 0,
|
| 155 |
"usd_per_1k_decisions": 0.0001,
|
| 156 |
"axes": {
|
| 157 |
-
"intelligence":
|
| 158 |
"intelligence_ci95": [
|
| 159 |
-
|
| 160 |
-
|
| 161 |
],
|
| 162 |
-
"calibration":
|
| 163 |
-
"speed":
|
| 164 |
"cost": 99.8
|
| 165 |
},
|
| 166 |
-
"capability_score":
|
| 167 |
-
"composite_score": 95.
|
| 168 |
"open_set_score": 100.0,
|
| 169 |
-
"sealed_set_score":
|
| 170 |
"calibration_details": {
|
| 171 |
-
"null_auroc": 0.
|
| 172 |
-
"null_ece": 0.
|
| 173 |
-
"null_brier": 0.
|
| 174 |
},
|
| 175 |
"latency": {
|
| 176 |
-
"mean_ms": 0.
|
| 177 |
-
"p50_ms": 0.
|
| 178 |
-
"p95_ms": 0.
|
| 179 |
-
"p99_ms": 0.
|
| 180 |
},
|
| 181 |
"by_request_type": {
|
| 182 |
"span_open": {
|
|
@@ -185,11 +193,11 @@
|
|
| 185 |
},
|
| 186 |
"math": {
|
| 187 |
"score": 100.0,
|
| 188 |
-
"n":
|
| 189 |
},
|
| 190 |
"choice": {
|
| 191 |
"score": 100.0,
|
| 192 |
-
"n":
|
| 193 |
},
|
| 194 |
"noul_open": {
|
| 195 |
"score": 100.0,
|
|
@@ -200,12 +208,12 @@
|
|
| 200 |
"n": 48
|
| 201 |
},
|
| 202 |
"span_sealed": {
|
| 203 |
-
"score":
|
| 204 |
-
"n":
|
| 205 |
},
|
| 206 |
"noul_sealed": {
|
| 207 |
-
"score":
|
| 208 |
-
"n":
|
| 209 |
}
|
| 210 |
},
|
| 211 |
"by_tier_open": {
|
|
@@ -224,12 +232,16 @@
|
|
| 224 |
},
|
| 225 |
"by_tier_sealed": {
|
| 226 |
"Standard": {
|
| 227 |
-
"score":
|
| 228 |
-
"n":
|
| 229 |
},
|
| 230 |
"Hard": {
|
| 231 |
-
"score":
|
| 232 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
}
|
| 234 |
},
|
| 235 |
"by_domain": {
|
|
@@ -253,6 +265,10 @@
|
|
| 253 |
"score": 23.3,
|
| 254 |
"n": 60
|
| 255 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
"adversarial_stress": {
|
| 257 |
"score": 100.0,
|
| 258 |
"n": 8
|
|
@@ -261,45 +277,45 @@
|
|
| 261 |
"official_rank": 2
|
| 262 |
},
|
| 263 |
{
|
| 264 |
-
"id": "nullspan-
|
| 265 |
-
"name": "nullspan-
|
| 266 |
"author": "gnumanth",
|
| 267 |
-
"system_type": "System One Encoder (5-Head)",
|
| 268 |
"open_weights": true,
|
| 269 |
-
"params_m":
|
| 270 |
-
"footprint_mb":
|
| 271 |
-
"base_model": "
|
| 272 |
-
"runtime": "PyTorch MPS/CPU FP32",
|
| 273 |
-
"input_rate_per_1m_usd": 0.
|
| 274 |
"output_rate_per_1m_usd": 0.0,
|
| 275 |
-
"n_observations":
|
| 276 |
-
"input_tokens_total":
|
| 277 |
"output_tokens_total": 0,
|
| 278 |
-
"usd_per_1k_decisions": 0.
|
| 279 |
"axes": {
|
| 280 |
-
"intelligence": 98.
|
| 281 |
"intelligence_ci95": [
|
| 282 |
-
|
| 283 |
-
99.
|
| 284 |
],
|
| 285 |
-
"calibration":
|
| 286 |
-
"speed":
|
| 287 |
-
"cost":
|
| 288 |
},
|
| 289 |
-
"capability_score":
|
| 290 |
-
"composite_score":
|
| 291 |
"open_set_score": 100.0,
|
| 292 |
-
"sealed_set_score":
|
| 293 |
"calibration_details": {
|
| 294 |
-
"null_auroc": 0.
|
| 295 |
-
"null_ece": 0.
|
| 296 |
-
"null_brier": 0.
|
| 297 |
},
|
| 298 |
"latency": {
|
| 299 |
-
"mean_ms":
|
| 300 |
-
"p50_ms":
|
| 301 |
-
"p95_ms":
|
| 302 |
-
"p99_ms":
|
| 303 |
},
|
| 304 |
"by_request_type": {
|
| 305 |
"span_open": {
|
|
@@ -308,11 +324,11 @@
|
|
| 308 |
},
|
| 309 |
"math": {
|
| 310 |
"score": 100.0,
|
| 311 |
-
"n":
|
| 312 |
},
|
| 313 |
"choice": {
|
| 314 |
"score": 100.0,
|
| 315 |
-
"n":
|
| 316 |
},
|
| 317 |
"noul_open": {
|
| 318 |
"score": 100.0,
|
|
@@ -323,12 +339,12 @@
|
|
| 323 |
"n": 48
|
| 324 |
},
|
| 325 |
"span_sealed": {
|
| 326 |
-
"score":
|
| 327 |
-
"n":
|
| 328 |
},
|
| 329 |
"noul_sealed": {
|
| 330 |
-
"score":
|
| 331 |
-
"n":
|
| 332 |
}
|
| 333 |
},
|
| 334 |
"by_tier_open": {
|
|
@@ -347,12 +363,16 @@
|
|
| 347 |
},
|
| 348 |
"by_tier_sealed": {
|
| 349 |
"Standard": {
|
| 350 |
-
"score":
|
| 351 |
-
"n":
|
| 352 |
},
|
| 353 |
"Hard": {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
"score": 100.0,
|
| 355 |
-
"n":
|
| 356 |
}
|
| 357 |
},
|
| 358 |
"by_domain": {
|
|
@@ -373,9 +393,13 @@
|
|
| 373 |
"n": 16
|
| 374 |
},
|
| 375 |
"squad_v2_open_qa": {
|
| 376 |
-
"score":
|
| 377 |
"n": 60
|
| 378 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 379 |
"adversarial_stress": {
|
| 380 |
"score": 100.0,
|
| 381 |
"n": 8
|
|
@@ -384,45 +408,45 @@
|
|
| 384 |
"official_rank": 3
|
| 385 |
},
|
| 386 |
{
|
| 387 |
-
"id": "nullspan-
|
| 388 |
-
"name": "nullspan-
|
| 389 |
"author": "gnumanth",
|
| 390 |
-
"system_type": "Quantized ONNX WASM/CPU",
|
| 391 |
"open_weights": true,
|
| 392 |
-
"params_m":
|
| 393 |
-
"footprint_mb":
|
| 394 |
"base_model": "gnumanth/nullspan (Int8 Dynamic)",
|
| 395 |
"runtime": "ONNX Runtime Int8 (Browser WASM / Edge)",
|
| 396 |
-
"input_rate_per_1m_usd": 0.
|
| 397 |
"output_rate_per_1m_usd": 0.0,
|
| 398 |
-
"n_observations":
|
| 399 |
-
"input_tokens_total":
|
| 400 |
"output_tokens_total": 0,
|
| 401 |
"usd_per_1k_decisions": 0.0008,
|
| 402 |
"axes": {
|
| 403 |
-
"intelligence":
|
| 404 |
"intelligence_ci95": [
|
| 405 |
-
96.
|
| 406 |
-
99.
|
| 407 |
],
|
| 408 |
-
"calibration":
|
| 409 |
-
"speed":
|
| 410 |
"cost": 84.2
|
| 411 |
},
|
| 412 |
-
"capability_score": 98.
|
| 413 |
-
"composite_score":
|
| 414 |
"open_set_score": 100.0,
|
| 415 |
-
"sealed_set_score":
|
| 416 |
"calibration_details": {
|
| 417 |
-
"null_auroc": 0.
|
| 418 |
-
"null_ece": 0.
|
| 419 |
-
"null_brier": 0.
|
| 420 |
},
|
| 421 |
"latency": {
|
| 422 |
-
"mean_ms":
|
| 423 |
-
"p50_ms":
|
| 424 |
-
"p95_ms":
|
| 425 |
-
"p99_ms":
|
| 426 |
},
|
| 427 |
"by_request_type": {
|
| 428 |
"span_open": {
|
|
@@ -431,11 +455,11 @@
|
|
| 431 |
},
|
| 432 |
"math": {
|
| 433 |
"score": 100.0,
|
| 434 |
-
"n":
|
| 435 |
},
|
| 436 |
"choice": {
|
| 437 |
"score": 100.0,
|
| 438 |
-
"n":
|
| 439 |
},
|
| 440 |
"noul_open": {
|
| 441 |
"score": 100.0,
|
|
@@ -446,12 +470,12 @@
|
|
| 446 |
"n": 48
|
| 447 |
},
|
| 448 |
"span_sealed": {
|
| 449 |
-
"score":
|
| 450 |
-
"n":
|
| 451 |
},
|
| 452 |
"noul_sealed": {
|
| 453 |
-
"score":
|
| 454 |
-
"n":
|
| 455 |
}
|
| 456 |
},
|
| 457 |
"by_tier_open": {
|
|
@@ -470,12 +494,16 @@
|
|
| 470 |
},
|
| 471 |
"by_tier_sealed": {
|
| 472 |
"Standard": {
|
| 473 |
-
"score":
|
| 474 |
-
"n":
|
| 475 |
},
|
| 476 |
"Hard": {
|
| 477 |
-
"score":
|
| 478 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
}
|
| 480 |
},
|
| 481 |
"by_domain": {
|
|
@@ -496,9 +524,13 @@
|
|
| 496 |
"n": 16
|
| 497 |
},
|
| 498 |
"squad_v2_open_qa": {
|
| 499 |
-
"score":
|
| 500 |
"n": 60
|
| 501 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 502 |
"adversarial_stress": {
|
| 503 |
"score": 100.0,
|
| 504 |
"n": 8
|
|
@@ -507,45 +539,45 @@
|
|
| 507 |
"official_rank": 4
|
| 508 |
},
|
| 509 |
{
|
| 510 |
-
"id": "nullspan-
|
| 511 |
-
"name": "nullspan-
|
| 512 |
"author": "gnumanth",
|
| 513 |
-
"system_type": "System One Encoder (ONNX)",
|
| 514 |
"open_weights": true,
|
| 515 |
-
"params_m":
|
| 516 |
-
"footprint_mb":
|
| 517 |
"base_model": "gnumanth/nullspan (ONNX Opset 17)",
|
| 518 |
"runtime": "ONNX Runtime FP32",
|
| 519 |
-
"input_rate_per_1m_usd": 0.
|
| 520 |
"output_rate_per_1m_usd": 0.0,
|
| 521 |
-
"n_observations":
|
| 522 |
-
"input_tokens_total":
|
| 523 |
"output_tokens_total": 0,
|
| 524 |
-
"usd_per_1k_decisions": 0.
|
| 525 |
"axes": {
|
| 526 |
-
"intelligence": 98.
|
| 527 |
"intelligence_ci95": [
|
| 528 |
-
|
| 529 |
-
99.
|
| 530 |
],
|
| 531 |
-
"calibration":
|
| 532 |
-
"speed":
|
| 533 |
-
"cost": 78.
|
| 534 |
},
|
| 535 |
-
"capability_score":
|
| 536 |
-
"composite_score":
|
| 537 |
"open_set_score": 100.0,
|
| 538 |
-
"sealed_set_score":
|
| 539 |
"calibration_details": {
|
| 540 |
-
"null_auroc": 0.
|
| 541 |
-
"null_ece": 0.
|
| 542 |
-
"null_brier": 0.
|
| 543 |
},
|
| 544 |
"latency": {
|
| 545 |
-
"mean_ms":
|
| 546 |
-
"p50_ms":
|
| 547 |
-
"p95_ms":
|
| 548 |
-
"p99_ms":
|
| 549 |
},
|
| 550 |
"by_request_type": {
|
| 551 |
"span_open": {
|
|
@@ -554,11 +586,11 @@
|
|
| 554 |
},
|
| 555 |
"math": {
|
| 556 |
"score": 100.0,
|
| 557 |
-
"n":
|
| 558 |
},
|
| 559 |
"choice": {
|
| 560 |
"score": 100.0,
|
| 561 |
-
"n":
|
| 562 |
},
|
| 563 |
"noul_open": {
|
| 564 |
"score": 100.0,
|
|
@@ -569,12 +601,12 @@
|
|
| 569 |
"n": 48
|
| 570 |
},
|
| 571 |
"span_sealed": {
|
| 572 |
-
"score":
|
| 573 |
-
"n":
|
| 574 |
},
|
| 575 |
"noul_sealed": {
|
| 576 |
-
"score":
|
| 577 |
-
"n":
|
| 578 |
}
|
| 579 |
},
|
| 580 |
"by_tier_open": {
|
|
@@ -593,12 +625,16 @@
|
|
| 593 |
},
|
| 594 |
"by_tier_sealed": {
|
| 595 |
"Standard": {
|
| 596 |
-
"score":
|
| 597 |
-
"n":
|
| 598 |
},
|
| 599 |
"Hard": {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 600 |
"score": 100.0,
|
| 601 |
-
"n":
|
| 602 |
}
|
| 603 |
},
|
| 604 |
"by_domain": {
|
|
@@ -619,9 +655,13 @@
|
|
| 619 |
"n": 16
|
| 620 |
},
|
| 621 |
"squad_v2_open_qa": {
|
| 622 |
-
"score":
|
| 623 |
"n": 60
|
| 624 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
"adversarial_stress": {
|
| 626 |
"score": 100.0,
|
| 627 |
"n": 8
|
|
@@ -745,6 +785,10 @@
|
|
| 745 |
"score": 62.6,
|
| 746 |
"n": 300
|
| 747 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 748 |
"adversarial_stress": {
|
| 749 |
"score": 54.3,
|
| 750 |
"n": 300
|
|
@@ -752,6 +796,137 @@
|
|
| 752 |
},
|
| 753 |
"official_rank": 6
|
| 754 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 755 |
{
|
| 756 |
"id": "basal-omni-1.0-ultra",
|
| 757 |
"name": "basal-omni-1.0-ultra (JevBench #1 Cap)",
|
|
@@ -858,7 +1033,7 @@
|
|
| 858 |
},
|
| 859 |
"cluster_throughput_comps": {
|
| 860 |
"score": 69.1,
|
| 861 |
-
"n":
|
| 862 |
},
|
| 863 |
"edge_traffic_forecast": {
|
| 864 |
"score": 66.9,
|
|
@@ -868,12 +1043,16 @@
|
|
| 868 |
"score": 76.9,
|
| 869 |
"n": 300
|
| 870 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 871 |
"adversarial_stress": {
|
| 872 |
"score": 74.0,
|
| 873 |
"n": 300
|
| 874 |
}
|
| 875 |
},
|
| 876 |
-
"official_rank":
|
| 877 |
},
|
| 878 |
{
|
| 879 |
"id": "jev-1.13.0-ref",
|
|
@@ -991,119 +1170,127 @@
|
|
| 991 |
"score": 61.6,
|
| 992 |
"n": 300
|
| 993 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 994 |
"adversarial_stress": {
|
| 995 |
"score": 58.9,
|
| 996 |
"n": 300
|
| 997 |
}
|
| 998 |
},
|
| 999 |
-
"official_rank":
|
| 1000 |
},
|
| 1001 |
{
|
| 1002 |
-
"id": "
|
| 1003 |
-
"name": "
|
| 1004 |
-
"author": "
|
| 1005 |
-
"system_type": "
|
| 1006 |
"open_weights": false,
|
| 1007 |
-
"params_m":
|
| 1008 |
"footprint_mb": null,
|
| 1009 |
-
"base_model": "
|
| 1010 |
-
"runtime": "
|
| 1011 |
-
"input_rate_per_1m_usd": 0.
|
| 1012 |
-
"output_rate_per_1m_usd": 0.
|
| 1013 |
-
"n_observations":
|
| 1014 |
-
"input_tokens_total":
|
| 1015 |
-
"output_tokens_total":
|
| 1016 |
-
"usd_per_1k_decisions": 0.
|
| 1017 |
"axes": {
|
| 1018 |
-
"intelligence":
|
| 1019 |
"intelligence_ci95": [
|
| 1020 |
-
|
| 1021 |
-
|
| 1022 |
],
|
| 1023 |
-
"calibration":
|
| 1024 |
-
"speed":
|
| 1025 |
-
"cost":
|
| 1026 |
},
|
| 1027 |
-
"capability_score":
|
| 1028 |
-
"composite_score":
|
| 1029 |
-
"open_set_score":
|
| 1030 |
-
"sealed_set_score":
|
| 1031 |
"calibration_details": {
|
| 1032 |
-
"null_auroc": 0.
|
| 1033 |
-
"null_ece": 0.
|
| 1034 |
-
"null_brier": 0.
|
| 1035 |
},
|
| 1036 |
"latency": {
|
| 1037 |
-
"mean_ms":
|
| 1038 |
-
"p50_ms":
|
| 1039 |
-
"p95_ms":
|
| 1040 |
-
"p99_ms":
|
| 1041 |
},
|
| 1042 |
"by_request_type": {
|
| 1043 |
"span_open": {
|
| 1044 |
-
"score":
|
| 1045 |
"n": 181
|
| 1046 |
},
|
| 1047 |
"math": {
|
| 1048 |
"score": 100.0,
|
| 1049 |
-
"n":
|
| 1050 |
},
|
| 1051 |
"choice": {
|
| 1052 |
"score": 100.0,
|
| 1053 |
-
"n":
|
| 1054 |
},
|
| 1055 |
"noul_open": {
|
| 1056 |
-
"score":
|
| 1057 |
"n": 27
|
| 1058 |
},
|
| 1059 |
"score": {
|
| 1060 |
-
"score":
|
| 1061 |
"n": 48
|
| 1062 |
},
|
| 1063 |
"span_sealed": {
|
| 1064 |
-
"score":
|
| 1065 |
-
"n":
|
| 1066 |
},
|
| 1067 |
"noul_sealed": {
|
| 1068 |
-
"score":
|
| 1069 |
-
"n":
|
| 1070 |
}
|
| 1071 |
},
|
| 1072 |
"by_tier_open": {
|
| 1073 |
"Easy": {
|
| 1074 |
-
"score":
|
| 1075 |
"n": 176
|
| 1076 |
},
|
| 1077 |
"Judge": {
|
| 1078 |
-
"score":
|
| 1079 |
"n": 96
|
| 1080 |
},
|
| 1081 |
"Standard": {
|
| 1082 |
-
"score":
|
| 1083 |
"n": 64
|
| 1084 |
}
|
| 1085 |
},
|
| 1086 |
"by_tier_sealed": {
|
| 1087 |
"Standard": {
|
| 1088 |
-
"score":
|
| 1089 |
-
"n":
|
| 1090 |
},
|
| 1091 |
"Hard": {
|
| 1092 |
-
"score":
|
| 1093 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1094 |
}
|
| 1095 |
},
|
| 1096 |
"by_domain": {
|
| 1097 |
"incident_report": {
|
| 1098 |
-
"score":
|
| 1099 |
"n": 160
|
| 1100 |
},
|
| 1101 |
"compute_ledger": {
|
| 1102 |
-
"score":
|
| 1103 |
"n": 144
|
| 1104 |
},
|
| 1105 |
"cluster_throughput_comps": {
|
| 1106 |
-
"score":
|
| 1107 |
"n": 16
|
| 1108 |
},
|
| 1109 |
"edge_traffic_forecast": {
|
|
@@ -1111,138 +1298,150 @@
|
|
| 1111 |
"n": 16
|
| 1112 |
},
|
| 1113 |
"squad_v2_open_qa": {
|
| 1114 |
-
"score":
|
| 1115 |
"n": 60
|
| 1116 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1117 |
"adversarial_stress": {
|
| 1118 |
"score": 87.5,
|
| 1119 |
"n": 8
|
| 1120 |
}
|
| 1121 |
},
|
| 1122 |
-
"official_rank":
|
| 1123 |
},
|
| 1124 |
{
|
| 1125 |
-
"id": "
|
| 1126 |
-
"name": "
|
| 1127 |
-
"author": "
|
| 1128 |
-
"system_type": "
|
| 1129 |
-
"open_weights":
|
| 1130 |
-
"params_m":
|
| 1131 |
-
"footprint_mb":
|
| 1132 |
-
"base_model": "
|
| 1133 |
-
"runtime": "
|
| 1134 |
-
"input_rate_per_1m_usd": 0.
|
| 1135 |
-
"output_rate_per_1m_usd": 0.
|
| 1136 |
-
"n_observations":
|
| 1137 |
-
"input_tokens_total":
|
| 1138 |
-
"output_tokens_total":
|
| 1139 |
-
"usd_per_1k_decisions": 0.
|
| 1140 |
"axes": {
|
| 1141 |
-
"intelligence":
|
| 1142 |
"intelligence_ci95": [
|
| 1143 |
-
|
| 1144 |
-
|
| 1145 |
],
|
| 1146 |
-
"calibration":
|
| 1147 |
-
"speed":
|
| 1148 |
-
"cost":
|
| 1149 |
-
},
|
| 1150 |
-
"capability_score":
|
| 1151 |
-
"composite_score":
|
| 1152 |
-
"open_set_score":
|
| 1153 |
-
"sealed_set_score":
|
| 1154 |
"calibration_details": {
|
| 1155 |
-
"null_auroc": 0.
|
| 1156 |
-
"null_ece": 0.
|
| 1157 |
-
"null_brier": 0.
|
| 1158 |
},
|
| 1159 |
"latency": {
|
| 1160 |
-
"mean_ms":
|
| 1161 |
-
"p50_ms":
|
| 1162 |
-
"p95_ms":
|
| 1163 |
-
"p99_ms":
|
| 1164 |
},
|
| 1165 |
"by_request_type": {
|
| 1166 |
"span_open": {
|
| 1167 |
-
"score":
|
| 1168 |
"n": 181
|
| 1169 |
},
|
| 1170 |
"math": {
|
| 1171 |
-
"score":
|
| 1172 |
-
"n":
|
| 1173 |
},
|
| 1174 |
"choice": {
|
| 1175 |
-
"score":
|
| 1176 |
-
"n":
|
| 1177 |
},
|
| 1178 |
"noul_open": {
|
| 1179 |
-
"score":
|
| 1180 |
"n": 27
|
| 1181 |
},
|
| 1182 |
"score": {
|
| 1183 |
-
"score":
|
| 1184 |
"n": 48
|
| 1185 |
},
|
| 1186 |
"span_sealed": {
|
| 1187 |
-
"score":
|
| 1188 |
-
"n":
|
| 1189 |
},
|
| 1190 |
"noul_sealed": {
|
| 1191 |
-
"score":
|
| 1192 |
-
"n":
|
| 1193 |
}
|
| 1194 |
},
|
| 1195 |
"by_tier_open": {
|
| 1196 |
"Easy": {
|
| 1197 |
-
"score":
|
| 1198 |
"n": 176
|
| 1199 |
},
|
| 1200 |
"Judge": {
|
| 1201 |
-
"score":
|
| 1202 |
"n": 96
|
| 1203 |
},
|
| 1204 |
"Standard": {
|
| 1205 |
-
"score":
|
| 1206 |
"n": 64
|
| 1207 |
}
|
| 1208 |
},
|
| 1209 |
"by_tier_sealed": {
|
| 1210 |
"Standard": {
|
| 1211 |
-
"score":
|
| 1212 |
-
"n":
|
| 1213 |
},
|
| 1214 |
"Hard": {
|
| 1215 |
-
"score": 78.
|
| 1216 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1217 |
}
|
| 1218 |
},
|
| 1219 |
"by_domain": {
|
| 1220 |
"incident_report": {
|
| 1221 |
-
"score":
|
| 1222 |
"n": 160
|
| 1223 |
},
|
| 1224 |
"compute_ledger": {
|
| 1225 |
-
"score":
|
| 1226 |
"n": 144
|
| 1227 |
},
|
| 1228 |
"cluster_throughput_comps": {
|
| 1229 |
-
"score":
|
| 1230 |
"n": 16
|
| 1231 |
},
|
| 1232 |
"edge_traffic_forecast": {
|
| 1233 |
-
"score":
|
| 1234 |
"n": 16
|
| 1235 |
},
|
| 1236 |
"squad_v2_open_qa": {
|
| 1237 |
"score": 80.0,
|
| 1238 |
"n": 60
|
| 1239 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1240 |
"adversarial_stress": {
|
| 1241 |
-
"score":
|
| 1242 |
"n": 8
|
| 1243 |
}
|
| 1244 |
},
|
| 1245 |
-
"official_rank":
|
| 1246 |
},
|
| 1247 |
{
|
| 1248 |
"id": "claude-haiku-4.5-json",
|
|
@@ -1256,34 +1455,34 @@
|
|
| 1256 |
"runtime": "Hosted Cloud API ( ~38 output tokens/field )",
|
| 1257 |
"input_rate_per_1m_usd": 0.8,
|
| 1258 |
"output_rate_per_1m_usd": 4.0,
|
| 1259 |
-
"n_observations":
|
| 1260 |
-
"input_tokens_total":
|
| 1261 |
-
"output_tokens_total":
|
| 1262 |
-
"usd_per_1k_decisions": 0.
|
| 1263 |
"axes": {
|
| 1264 |
-
"intelligence":
|
| 1265 |
"intelligence_ci95": [
|
| 1266 |
-
|
| 1267 |
-
|
| 1268 |
],
|
| 1269 |
-
"calibration": 88.
|
| 1270 |
"speed": 41.8,
|
| 1271 |
-
"cost":
|
| 1272 |
},
|
| 1273 |
-
"capability_score": 92.
|
| 1274 |
-
"composite_score":
|
| 1275 |
-
"open_set_score":
|
| 1276 |
-
"sealed_set_score":
|
| 1277 |
"calibration_details": {
|
| 1278 |
-
"null_auroc": 0.
|
| 1279 |
-
"null_ece": 0.
|
| 1280 |
-
"null_brier": 0.
|
| 1281 |
},
|
| 1282 |
"latency": {
|
| 1283 |
-
"mean_ms":
|
| 1284 |
-
"p50_ms":
|
| 1285 |
-
"p95_ms":
|
| 1286 |
-
"p99_ms": 727.
|
| 1287 |
},
|
| 1288 |
"by_request_type": {
|
| 1289 |
"span_open": {
|
|
@@ -1292,14 +1491,14 @@
|
|
| 1292 |
},
|
| 1293 |
"math": {
|
| 1294 |
"score": 100.0,
|
| 1295 |
-
"n":
|
| 1296 |
},
|
| 1297 |
"choice": {
|
| 1298 |
"score": 100.0,
|
| 1299 |
-
"n":
|
| 1300 |
},
|
| 1301 |
"noul_open": {
|
| 1302 |
-
"score":
|
| 1303 |
"n": 27
|
| 1304 |
},
|
| 1305 |
"score": {
|
|
@@ -1307,17 +1506,17 @@
|
|
| 1307 |
"n": 48
|
| 1308 |
},
|
| 1309 |
"span_sealed": {
|
| 1310 |
-
"score":
|
| 1311 |
-
"n":
|
| 1312 |
},
|
| 1313 |
"noul_sealed": {
|
| 1314 |
-
"score":
|
| 1315 |
-
"n":
|
| 1316 |
}
|
| 1317 |
},
|
| 1318 |
"by_tier_open": {
|
| 1319 |
"Easy": {
|
| 1320 |
-
"score":
|
| 1321 |
"n": 176
|
| 1322 |
},
|
| 1323 |
"Judge": {
|
|
@@ -1331,21 +1530,25 @@
|
|
| 1331 |
},
|
| 1332 |
"by_tier_sealed": {
|
| 1333 |
"Standard": {
|
| 1334 |
-
"score":
|
| 1335 |
-
"n":
|
| 1336 |
},
|
| 1337 |
"Hard": {
|
| 1338 |
-
"score":
|
| 1339 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1340 |
}
|
| 1341 |
},
|
| 1342 |
"by_domain": {
|
| 1343 |
"incident_report": {
|
| 1344 |
-
"score":
|
| 1345 |
"n": 160
|
| 1346 |
},
|
| 1347 |
"compute_ledger": {
|
| 1348 |
-
"score":
|
| 1349 |
"n": 144
|
| 1350 |
},
|
| 1351 |
"cluster_throughput_comps": {
|
|
@@ -1357,15 +1560,19 @@
|
|
| 1357 |
"n": 16
|
| 1358 |
},
|
| 1359 |
"squad_v2_open_qa": {
|
| 1360 |
-
"score":
|
| 1361 |
"n": 60
|
| 1362 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1363 |
"adversarial_stress": {
|
| 1364 |
-
"score":
|
| 1365 |
"n": 8
|
| 1366 |
}
|
| 1367 |
},
|
| 1368 |
-
"official_rank":
|
| 1369 |
},
|
| 1370 |
{
|
| 1371 |
"id": "bge-small-en-v1.5",
|
|
@@ -1379,34 +1586,34 @@
|
|
| 1379 |
"runtime": "PyTorch MPS/CPU",
|
| 1380 |
"input_rate_per_1m_usd": 0.01,
|
| 1381 |
"output_rate_per_1m_usd": 0.0,
|
| 1382 |
-
"n_observations":
|
| 1383 |
-
"input_tokens_total":
|
| 1384 |
"output_tokens_total": 0,
|
| 1385 |
-
"usd_per_1k_decisions": 0.
|
| 1386 |
"axes": {
|
| 1387 |
-
"intelligence":
|
| 1388 |
"intelligence_ci95": [
|
| 1389 |
-
|
| 1390 |
-
|
| 1391 |
],
|
| 1392 |
-
"calibration": 89.
|
| 1393 |
-
"speed": 64.
|
| 1394 |
-
"cost":
|
| 1395 |
},
|
| 1396 |
-
"capability_score":
|
| 1397 |
-
"composite_score":
|
| 1398 |
"open_set_score": 20.3,
|
| 1399 |
-
"sealed_set_score": 3.
|
| 1400 |
"calibration_details": {
|
| 1401 |
-
"null_auroc": 0.
|
| 1402 |
-
"null_ece": 0.
|
| 1403 |
-
"null_brier": 0.
|
| 1404 |
},
|
| 1405 |
"latency": {
|
| 1406 |
-
"mean_ms":
|
| 1407 |
-
"p50_ms":
|
| 1408 |
-
"p95_ms":
|
| 1409 |
-
"p99_ms":
|
| 1410 |
},
|
| 1411 |
"by_request_type": {
|
| 1412 |
"span_open": {
|
|
@@ -1415,11 +1622,11 @@
|
|
| 1415 |
},
|
| 1416 |
"math": {
|
| 1417 |
"score": 0.0,
|
| 1418 |
-
"n":
|
| 1419 |
},
|
| 1420 |
"choice": {
|
| 1421 |
-
"score":
|
| 1422 |
-
"n":
|
| 1423 |
},
|
| 1424 |
"noul_open": {
|
| 1425 |
"score": 44.4,
|
|
@@ -1430,12 +1637,12 @@
|
|
| 1430 |
"n": 48
|
| 1431 |
},
|
| 1432 |
"span_sealed": {
|
| 1433 |
-
"score":
|
| 1434 |
-
"n":
|
| 1435 |
},
|
| 1436 |
"noul_sealed": {
|
| 1437 |
-
"score":
|
| 1438 |
-
"n":
|
| 1439 |
}
|
| 1440 |
},
|
| 1441 |
"by_tier_open": {
|
|
@@ -1454,12 +1661,16 @@
|
|
| 1454 |
},
|
| 1455 |
"by_tier_sealed": {
|
| 1456 |
"Standard": {
|
| 1457 |
-
"score":
|
| 1458 |
-
"n":
|
| 1459 |
},
|
| 1460 |
"Hard": {
|
| 1461 |
-
"score":
|
| 1462 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1463 |
}
|
| 1464 |
},
|
| 1465 |
"by_domain": {
|
|
@@ -1483,12 +1694,16 @@
|
|
| 1483 |
"score": 0.0,
|
| 1484 |
"n": 60
|
| 1485 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1486 |
"adversarial_stress": {
|
| 1487 |
"score": 31.2,
|
| 1488 |
"n": 8
|
| 1489 |
}
|
| 1490 |
},
|
| 1491 |
-
"official_rank":
|
| 1492 |
},
|
| 1493 |
{
|
| 1494 |
"id": "distilbert-base-control",
|
|
@@ -1502,34 +1717,34 @@
|
|
| 1502 |
"runtime": "PyTorch MPS/CPU",
|
| 1503 |
"input_rate_per_1m_usd": 0.015,
|
| 1504 |
"output_rate_per_1m_usd": 0.0,
|
| 1505 |
-
"n_observations":
|
| 1506 |
-
"input_tokens_total":
|
| 1507 |
"output_tokens_total": 0,
|
| 1508 |
-
"usd_per_1k_decisions": 0.
|
| 1509 |
"axes": {
|
| 1510 |
-
"intelligence":
|
| 1511 |
"intelligence_ci95": [
|
| 1512 |
-
|
| 1513 |
-
|
| 1514 |
],
|
| 1515 |
-
"calibration":
|
| 1516 |
-
"speed":
|
| 1517 |
-
"cost":
|
| 1518 |
},
|
| 1519 |
-
"capability_score":
|
| 1520 |
-
"composite_score":
|
| 1521 |
"open_set_score": 20.2,
|
| 1522 |
-
"sealed_set_score":
|
| 1523 |
"calibration_details": {
|
| 1524 |
-
"null_auroc": 0.
|
| 1525 |
-
"null_ece": 0.
|
| 1526 |
-
"null_brier": 0.
|
| 1527 |
},
|
| 1528 |
"latency": {
|
| 1529 |
-
"mean_ms":
|
| 1530 |
-
"p50_ms": 6.
|
| 1531 |
-
"p95_ms":
|
| 1532 |
-
"p99_ms":
|
| 1533 |
},
|
| 1534 |
"by_request_type": {
|
| 1535 |
"span_open": {
|
|
@@ -1538,11 +1753,11 @@
|
|
| 1538 |
},
|
| 1539 |
"math": {
|
| 1540 |
"score": 0.0,
|
| 1541 |
-
"n":
|
| 1542 |
},
|
| 1543 |
"choice": {
|
| 1544 |
-
"score":
|
| 1545 |
-
"n":
|
| 1546 |
},
|
| 1547 |
"noul_open": {
|
| 1548 |
"score": 0.0,
|
|
@@ -1553,12 +1768,12 @@
|
|
| 1553 |
"n": 48
|
| 1554 |
},
|
| 1555 |
"span_sealed": {
|
| 1556 |
-
"score":
|
| 1557 |
-
"n":
|
| 1558 |
},
|
| 1559 |
"noul_sealed": {
|
| 1560 |
"score": 0.0,
|
| 1561 |
-
"n":
|
| 1562 |
}
|
| 1563 |
},
|
| 1564 |
"by_tier_open": {
|
|
@@ -1577,12 +1792,16 @@
|
|
| 1577 |
},
|
| 1578 |
"by_tier_sealed": {
|
| 1579 |
"Standard": {
|
| 1580 |
-
"score":
|
| 1581 |
-
"n":
|
| 1582 |
},
|
| 1583 |
"Hard": {
|
| 1584 |
-
"score": 2.
|
| 1585 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1586 |
}
|
| 1587 |
},
|
| 1588 |
"by_domain": {
|
|
@@ -1606,12 +1825,16 @@
|
|
| 1606 |
"score": 1.7,
|
| 1607 |
"n": 60
|
| 1608 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1609 |
"adversarial_stress": {
|
| 1610 |
"score": 12.5,
|
| 1611 |
"n": 8
|
| 1612 |
}
|
| 1613 |
},
|
| 1614 |
-
"official_rank":
|
| 1615 |
},
|
| 1616 |
{
|
| 1617 |
"id": "all-MiniLM-L12-v2",
|
|
@@ -1625,34 +1848,34 @@
|
|
| 1625 |
"runtime": "PyTorch MPS/CPU",
|
| 1626 |
"input_rate_per_1m_usd": 0.01,
|
| 1627 |
"output_rate_per_1m_usd": 0.0,
|
| 1628 |
-
"n_observations":
|
| 1629 |
-
"input_tokens_total":
|
| 1630 |
"output_tokens_total": 0,
|
| 1631 |
-
"usd_per_1k_decisions": 0.
|
| 1632 |
"axes": {
|
| 1633 |
-
"intelligence":
|
| 1634 |
"intelligence_ci95": [
|
| 1635 |
-
|
| 1636 |
-
|
| 1637 |
],
|
| 1638 |
-
"calibration":
|
| 1639 |
-
"speed":
|
| 1640 |
-
"cost":
|
| 1641 |
},
|
| 1642 |
-
"capability_score": 44.
|
| 1643 |
-
"composite_score":
|
| 1644 |
"open_set_score": 17.2,
|
| 1645 |
-
"sealed_set_score":
|
| 1646 |
"calibration_details": {
|
| 1647 |
-
"null_auroc": 0.
|
| 1648 |
-
"null_ece": 0.
|
| 1649 |
-
"null_brier": 0.
|
| 1650 |
},
|
| 1651 |
"latency": {
|
| 1652 |
-
"mean_ms":
|
| 1653 |
-
"p50_ms":
|
| 1654 |
-
"p95_ms":
|
| 1655 |
-
"p99_ms":
|
| 1656 |
},
|
| 1657 |
"by_request_type": {
|
| 1658 |
"span_open": {
|
|
@@ -1661,11 +1884,11 @@
|
|
| 1661 |
},
|
| 1662 |
"math": {
|
| 1663 |
"score": 0.0,
|
| 1664 |
-
"n":
|
| 1665 |
},
|
| 1666 |
"choice": {
|
| 1667 |
-
"score":
|
| 1668 |
-
"n":
|
| 1669 |
},
|
| 1670 |
"noul_open": {
|
| 1671 |
"score": 100.0,
|
|
@@ -1676,12 +1899,12 @@
|
|
| 1676 |
"n": 48
|
| 1677 |
},
|
| 1678 |
"span_sealed": {
|
| 1679 |
-
"score":
|
| 1680 |
-
"n":
|
| 1681 |
},
|
| 1682 |
"noul_sealed": {
|
| 1683 |
-
"score":
|
| 1684 |
-
"n":
|
| 1685 |
}
|
| 1686 |
},
|
| 1687 |
"by_tier_open": {
|
|
@@ -1700,12 +1923,16 @@
|
|
| 1700 |
},
|
| 1701 |
"by_tier_sealed": {
|
| 1702 |
"Standard": {
|
| 1703 |
-
"score": 0.
|
| 1704 |
-
"n":
|
| 1705 |
},
|
| 1706 |
"Hard": {
|
| 1707 |
-
"score":
|
| 1708 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1709 |
}
|
| 1710 |
},
|
| 1711 |
"by_domain": {
|
|
@@ -1729,12 +1956,16 @@
|
|
| 1729 |
"score": 1.7,
|
| 1730 |
"n": 60
|
| 1731 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1732 |
"adversarial_stress": {
|
| 1733 |
"score": 31.2,
|
| 1734 |
"n": 8
|
| 1735 |
}
|
| 1736 |
},
|
| 1737 |
-
"official_rank":
|
| 1738 |
},
|
| 1739 |
{
|
| 1740 |
"id": "all-mpnet-base-v2",
|
|
@@ -1748,34 +1979,34 @@
|
|
| 1748 |
"runtime": "PyTorch MPS/CPU",
|
| 1749 |
"input_rate_per_1m_usd": 0.022,
|
| 1750 |
"output_rate_per_1m_usd": 0.0,
|
| 1751 |
-
"n_observations":
|
| 1752 |
-
"input_tokens_total":
|
| 1753 |
"output_tokens_total": 0,
|
| 1754 |
-
"usd_per_1k_decisions": 0.
|
| 1755 |
"axes": {
|
| 1756 |
-
"intelligence":
|
| 1757 |
"intelligence_ci95": [
|
| 1758 |
10.9,
|
| 1759 |
-
|
| 1760 |
],
|
| 1761 |
-
"calibration":
|
| 1762 |
"speed": 62.8,
|
| 1763 |
-
"cost":
|
| 1764 |
},
|
| 1765 |
-
"capability_score":
|
| 1766 |
-
"composite_score":
|
| 1767 |
"open_set_score": 16.2,
|
| 1768 |
-
"sealed_set_score":
|
| 1769 |
"calibration_details": {
|
| 1770 |
-
"null_auroc": 0.
|
| 1771 |
-
"null_ece": 0.
|
| 1772 |
-
"null_brier": 0.
|
| 1773 |
},
|
| 1774 |
"latency": {
|
| 1775 |
-
"mean_ms":
|
| 1776 |
-
"p50_ms": 11.
|
| 1777 |
-
"p95_ms":
|
| 1778 |
-
"p99_ms":
|
| 1779 |
},
|
| 1780 |
"by_request_type": {
|
| 1781 |
"span_open": {
|
|
@@ -1784,11 +2015,11 @@
|
|
| 1784 |
},
|
| 1785 |
"math": {
|
| 1786 |
"score": 0.0,
|
| 1787 |
-
"n":
|
| 1788 |
},
|
| 1789 |
"choice": {
|
| 1790 |
-
"score":
|
| 1791 |
-
"n":
|
| 1792 |
},
|
| 1793 |
"noul_open": {
|
| 1794 |
"score": 100.0,
|
|
@@ -1799,12 +2030,12 @@
|
|
| 1799 |
"n": 48
|
| 1800 |
},
|
| 1801 |
"span_sealed": {
|
| 1802 |
-
"score":
|
| 1803 |
-
"n":
|
| 1804 |
},
|
| 1805 |
"noul_sealed": {
|
| 1806 |
-
"score":
|
| 1807 |
-
"n":
|
| 1808 |
}
|
| 1809 |
},
|
| 1810 |
"by_tier_open": {
|
|
@@ -1823,12 +2054,16 @@
|
|
| 1823 |
},
|
| 1824 |
"by_tier_sealed": {
|
| 1825 |
"Standard": {
|
| 1826 |
-
"score": 0.
|
| 1827 |
-
"n":
|
| 1828 |
},
|
| 1829 |
"Hard": {
|
| 1830 |
-
"score":
|
| 1831 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1832 |
}
|
| 1833 |
},
|
| 1834 |
"by_domain": {
|
|
@@ -1852,12 +2087,16 @@
|
|
| 1852 |
"score": 0.0,
|
| 1853 |
"n": 60
|
| 1854 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1855 |
"adversarial_stress": {
|
| 1856 |
"score": 31.2,
|
| 1857 |
"n": 8
|
| 1858 |
}
|
| 1859 |
},
|
| 1860 |
-
"official_rank":
|
| 1861 |
},
|
| 1862 |
{
|
| 1863 |
"id": "paraphrase-albert-small-v2",
|
|
@@ -1871,34 +2110,34 @@
|
|
| 1871 |
"runtime": "PyTorch MPS/CPU",
|
| 1872 |
"input_rate_per_1m_usd": 0.005,
|
| 1873 |
"output_rate_per_1m_usd": 0.0,
|
| 1874 |
-
"n_observations":
|
| 1875 |
-
"input_tokens_total":
|
| 1876 |
"output_tokens_total": 0,
|
| 1877 |
-
"usd_per_1k_decisions": 0.
|
| 1878 |
"axes": {
|
| 1879 |
-
"intelligence":
|
| 1880 |
"intelligence_ci95": [
|
| 1881 |
-
10.
|
| 1882 |
-
|
| 1883 |
],
|
| 1884 |
-
"calibration":
|
| 1885 |
-
"speed":
|
| 1886 |
-
"cost":
|
| 1887 |
},
|
| 1888 |
-
"capability_score":
|
| 1889 |
-
"composite_score":
|
| 1890 |
"open_set_score": 15.3,
|
| 1891 |
-
"sealed_set_score":
|
| 1892 |
"calibration_details": {
|
| 1893 |
-
"null_auroc": 0.
|
| 1894 |
-
"null_ece": 0.
|
| 1895 |
-
"null_brier": 0.
|
| 1896 |
},
|
| 1897 |
"latency": {
|
| 1898 |
-
"mean_ms":
|
| 1899 |
-
"p50_ms":
|
| 1900 |
-
"p95_ms":
|
| 1901 |
-
"p99_ms":
|
| 1902 |
},
|
| 1903 |
"by_request_type": {
|
| 1904 |
"span_open": {
|
|
@@ -1906,12 +2145,12 @@
|
|
| 1906 |
"n": 181
|
| 1907 |
},
|
| 1908 |
"math": {
|
| 1909 |
-
"score":
|
| 1910 |
-
"n":
|
| 1911 |
},
|
| 1912 |
"choice": {
|
| 1913 |
-
"score":
|
| 1914 |
-
"n":
|
| 1915 |
},
|
| 1916 |
"noul_open": {
|
| 1917 |
"score": 51.9,
|
|
@@ -1922,12 +2161,12 @@
|
|
| 1922 |
"n": 48
|
| 1923 |
},
|
| 1924 |
"span_sealed": {
|
| 1925 |
-
"score":
|
| 1926 |
-
"n":
|
| 1927 |
},
|
| 1928 |
"noul_sealed": {
|
| 1929 |
-
"score":
|
| 1930 |
-
"n":
|
| 1931 |
}
|
| 1932 |
},
|
| 1933 |
"by_tier_open": {
|
|
@@ -1946,12 +2185,16 @@
|
|
| 1946 |
},
|
| 1947 |
"by_tier_sealed": {
|
| 1948 |
"Standard": {
|
| 1949 |
-
"score":
|
| 1950 |
-
"n":
|
| 1951 |
},
|
| 1952 |
"Hard": {
|
| 1953 |
-
"score":
|
| 1954 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1955 |
}
|
| 1956 |
},
|
| 1957 |
"by_domain": {
|
|
@@ -1975,12 +2218,16 @@
|
|
| 1975 |
"score": 0.0,
|
| 1976 |
"n": 60
|
| 1977 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1978 |
"adversarial_stress": {
|
| 1979 |
"score": 18.8,
|
| 1980 |
"n": 8
|
| 1981 |
}
|
| 1982 |
},
|
| 1983 |
-
"official_rank":
|
| 1984 |
},
|
| 1985 |
{
|
| 1986 |
"id": "paraphrase-MiniLM-L3-v2",
|
|
@@ -1994,34 +2241,34 @@
|
|
| 1994 |
"runtime": "PyTorch MPS/CPU",
|
| 1995 |
"input_rate_per_1m_usd": 0.005,
|
| 1996 |
"output_rate_per_1m_usd": 0.0,
|
| 1997 |
-
"n_observations":
|
| 1998 |
-
"input_tokens_total":
|
| 1999 |
"output_tokens_total": 0,
|
| 2000 |
-
"usd_per_1k_decisions": 0.
|
| 2001 |
"axes": {
|
| 2002 |
-
"intelligence": 11.
|
| 2003 |
"intelligence_ci95": [
|
| 2004 |
-
8.
|
| 2005 |
-
|
| 2006 |
],
|
| 2007 |
-
"calibration":
|
| 2008 |
-
"speed":
|
| 2009 |
-
"cost":
|
| 2010 |
},
|
| 2011 |
-
"capability_score":
|
| 2012 |
-
"composite_score":
|
| 2013 |
"open_set_score": 12.9,
|
| 2014 |
-
"sealed_set_score":
|
| 2015 |
"calibration_details": {
|
| 2016 |
-
"null_auroc": 0.
|
| 2017 |
-
"null_ece": 0.
|
| 2018 |
-
"null_brier": 0.
|
| 2019 |
},
|
| 2020 |
"latency": {
|
| 2021 |
-
"mean_ms":
|
| 2022 |
-
"p50_ms":
|
| 2023 |
-
"p95_ms":
|
| 2024 |
-
"p99_ms":
|
| 2025 |
},
|
| 2026 |
"by_request_type": {
|
| 2027 |
"span_open": {
|
|
@@ -2030,11 +2277,11 @@
|
|
| 2030 |
},
|
| 2031 |
"math": {
|
| 2032 |
"score": 0.0,
|
| 2033 |
-
"n":
|
| 2034 |
},
|
| 2035 |
"choice": {
|
| 2036 |
-
"score":
|
| 2037 |
-
"n":
|
| 2038 |
},
|
| 2039 |
"noul_open": {
|
| 2040 |
"score": 44.4,
|
|
@@ -2045,12 +2292,12 @@
|
|
| 2045 |
"n": 48
|
| 2046 |
},
|
| 2047 |
"span_sealed": {
|
| 2048 |
-
"score":
|
| 2049 |
-
"n":
|
| 2050 |
},
|
| 2051 |
"noul_sealed": {
|
| 2052 |
-
"score":
|
| 2053 |
-
"n":
|
| 2054 |
}
|
| 2055 |
},
|
| 2056 |
"by_tier_open": {
|
|
@@ -2069,12 +2316,16 @@
|
|
| 2069 |
},
|
| 2070 |
"by_tier_sealed": {
|
| 2071 |
"Standard": {
|
| 2072 |
-
"score": 0.
|
| 2073 |
-
"n":
|
| 2074 |
},
|
| 2075 |
"Hard": {
|
| 2076 |
-
"score":
|
| 2077 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2078 |
}
|
| 2079 |
},
|
| 2080 |
"by_domain": {
|
|
@@ -2098,12 +2349,16 @@
|
|
| 2098 |
"score": 1.7,
|
| 2099 |
"n": 60
|
| 2100 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2101 |
"adversarial_stress": {
|
| 2102 |
"score": 18.8,
|
| 2103 |
"n": 8
|
| 2104 |
}
|
| 2105 |
},
|
| 2106 |
-
"official_rank":
|
| 2107 |
},
|
| 2108 |
{
|
| 2109 |
"id": "all-MiniLM-L6-v2",
|
|
@@ -2117,34 +2372,34 @@
|
|
| 2117 |
"runtime": "PyTorch MPS/CPU",
|
| 2118 |
"input_rate_per_1m_usd": 0.007,
|
| 2119 |
"output_rate_per_1m_usd": 0.0,
|
| 2120 |
-
"n_observations":
|
| 2121 |
-
"input_tokens_total":
|
| 2122 |
"output_tokens_total": 0,
|
| 2123 |
-
"usd_per_1k_decisions": 0.
|
| 2124 |
"axes": {
|
| 2125 |
-
"intelligence":
|
| 2126 |
"intelligence_ci95": [
|
| 2127 |
-
7.
|
| 2128 |
-
12.
|
| 2129 |
],
|
| 2130 |
-
"calibration":
|
| 2131 |
-
"speed":
|
| 2132 |
-
"cost":
|
| 2133 |
},
|
| 2134 |
-
"capability_score":
|
| 2135 |
-
"composite_score": 28.
|
| 2136 |
"open_set_score": 11.2,
|
| 2137 |
-
"sealed_set_score":
|
| 2138 |
"calibration_details": {
|
| 2139 |
-
"null_auroc": 0.
|
| 2140 |
-
"null_ece": 0.
|
| 2141 |
-
"null_brier": 0.
|
| 2142 |
},
|
| 2143 |
"latency": {
|
| 2144 |
-
"mean_ms":
|
| 2145 |
-
"p50_ms":
|
| 2146 |
-
"p95_ms":
|
| 2147 |
-
"p99_ms":
|
| 2148 |
},
|
| 2149 |
"by_request_type": {
|
| 2150 |
"span_open": {
|
|
@@ -2153,11 +2408,11 @@
|
|
| 2153 |
},
|
| 2154 |
"math": {
|
| 2155 |
"score": 0.0,
|
| 2156 |
-
"n":
|
| 2157 |
},
|
| 2158 |
"choice": {
|
| 2159 |
-
"score":
|
| 2160 |
-
"n":
|
| 2161 |
},
|
| 2162 |
"noul_open": {
|
| 2163 |
"score": 44.4,
|
|
@@ -2168,12 +2423,12 @@
|
|
| 2168 |
"n": 48
|
| 2169 |
},
|
| 2170 |
"span_sealed": {
|
| 2171 |
-
"score":
|
| 2172 |
-
"n":
|
| 2173 |
},
|
| 2174 |
"noul_sealed": {
|
| 2175 |
-
"score":
|
| 2176 |
-
"n":
|
| 2177 |
}
|
| 2178 |
},
|
| 2179 |
"by_tier_open": {
|
|
@@ -2192,12 +2447,16 @@
|
|
| 2192 |
},
|
| 2193 |
"by_tier_sealed": {
|
| 2194 |
"Standard": {
|
| 2195 |
-
"score": 0.
|
| 2196 |
-
"n":
|
| 2197 |
},
|
| 2198 |
"Hard": {
|
| 2199 |
-
"score":
|
| 2200 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2201 |
}
|
| 2202 |
},
|
| 2203 |
"by_domain": {
|
|
@@ -2221,55 +2480,59 @@
|
|
| 2221 |
"score": 1.7,
|
| 2222 |
"n": 60
|
| 2223 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2224 |
"adversarial_stress": {
|
| 2225 |
"score": 18.8,
|
| 2226 |
"n": 8
|
| 2227 |
}
|
| 2228 |
},
|
| 2229 |
-
"official_rank":
|
| 2230 |
}
|
| 2231 |
],
|
| 2232 |
"tiers": {
|
| 2233 |
"tier_1a_pytorch_fp32": {
|
| 2234 |
-
"id": "nullspan-
|
| 2235 |
-
"name": "nullspan-
|
| 2236 |
"author": "gnumanth",
|
| 2237 |
-
"system_type": "System One Encoder (5-Head)",
|
| 2238 |
"open_weights": true,
|
| 2239 |
-
"params_m":
|
| 2240 |
-
"footprint_mb":
|
| 2241 |
-
"base_model": "
|
| 2242 |
-
"runtime": "PyTorch MPS/CPU FP32",
|
| 2243 |
-
"input_rate_per_1m_usd": 0.
|
| 2244 |
"output_rate_per_1m_usd": 0.0,
|
| 2245 |
-
"n_observations":
|
| 2246 |
-
"input_tokens_total":
|
| 2247 |
"output_tokens_total": 0,
|
| 2248 |
-
"usd_per_1k_decisions": 0.
|
| 2249 |
"axes": {
|
| 2250 |
-
"intelligence": 98.
|
| 2251 |
"intelligence_ci95": [
|
| 2252 |
-
|
| 2253 |
-
99.
|
| 2254 |
],
|
| 2255 |
-
"calibration":
|
| 2256 |
-
"speed":
|
| 2257 |
-
"cost":
|
| 2258 |
},
|
| 2259 |
-
"capability_score":
|
| 2260 |
-
"composite_score":
|
| 2261 |
"open_set_score": 100.0,
|
| 2262 |
-
"sealed_set_score":
|
| 2263 |
"calibration_details": {
|
| 2264 |
-
"null_auroc": 0.
|
| 2265 |
-
"null_ece": 0.
|
| 2266 |
-
"null_brier": 0.
|
| 2267 |
},
|
| 2268 |
"latency": {
|
| 2269 |
-
"mean_ms":
|
| 2270 |
-
"p50_ms":
|
| 2271 |
-
"p95_ms":
|
| 2272 |
-
"p99_ms":
|
| 2273 |
},
|
| 2274 |
"by_request_type": {
|
| 2275 |
"span_open": {
|
|
@@ -2278,11 +2541,11 @@
|
|
| 2278 |
},
|
| 2279 |
"math": {
|
| 2280 |
"score": 100.0,
|
| 2281 |
-
"n":
|
| 2282 |
},
|
| 2283 |
"choice": {
|
| 2284 |
"score": 100.0,
|
| 2285 |
-
"n":
|
| 2286 |
},
|
| 2287 |
"noul_open": {
|
| 2288 |
"score": 100.0,
|
|
@@ -2293,12 +2556,12 @@
|
|
| 2293 |
"n": 48
|
| 2294 |
},
|
| 2295 |
"span_sealed": {
|
| 2296 |
-
"score":
|
| 2297 |
-
"n":
|
| 2298 |
},
|
| 2299 |
"noul_sealed": {
|
| 2300 |
-
"score":
|
| 2301 |
-
"n":
|
| 2302 |
}
|
| 2303 |
},
|
| 2304 |
"by_tier_open": {
|
|
@@ -2317,12 +2580,16 @@
|
|
| 2317 |
},
|
| 2318 |
"by_tier_sealed": {
|
| 2319 |
"Standard": {
|
| 2320 |
-
"score":
|
| 2321 |
-
"n":
|
| 2322 |
},
|
| 2323 |
"Hard": {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2324 |
"score": 100.0,
|
| 2325 |
-
"n":
|
| 2326 |
}
|
| 2327 |
},
|
| 2328 |
"by_domain": {
|
|
@@ -2343,9 +2610,13 @@
|
|
| 2343 |
"n": 16
|
| 2344 |
},
|
| 2345 |
"squad_v2_open_qa": {
|
| 2346 |
-
"score":
|
| 2347 |
"n": 60
|
| 2348 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2349 |
"adversarial_stress": {
|
| 2350 |
"score": 100.0,
|
| 2351 |
"n": 8
|
|
@@ -2354,45 +2625,45 @@
|
|
| 2354 |
"official_rank": 3
|
| 2355 |
},
|
| 2356 |
"tier_1b_onnx_int8": {
|
| 2357 |
-
"id": "nullspan-
|
| 2358 |
-
"name": "nullspan-
|
| 2359 |
"author": "gnumanth",
|
| 2360 |
-
"system_type": "Quantized ONNX WASM/CPU",
|
| 2361 |
"open_weights": true,
|
| 2362 |
-
"params_m":
|
| 2363 |
-
"footprint_mb":
|
| 2364 |
"base_model": "gnumanth/nullspan (Int8 Dynamic)",
|
| 2365 |
"runtime": "ONNX Runtime Int8 (Browser WASM / Edge)",
|
| 2366 |
-
"input_rate_per_1m_usd": 0.
|
| 2367 |
"output_rate_per_1m_usd": 0.0,
|
| 2368 |
-
"n_observations":
|
| 2369 |
-
"input_tokens_total":
|
| 2370 |
"output_tokens_total": 0,
|
| 2371 |
"usd_per_1k_decisions": 0.0008,
|
| 2372 |
"axes": {
|
| 2373 |
-
"intelligence":
|
| 2374 |
"intelligence_ci95": [
|
| 2375 |
-
96.
|
| 2376 |
-
99.
|
| 2377 |
],
|
| 2378 |
-
"calibration":
|
| 2379 |
-
"speed":
|
| 2380 |
"cost": 84.2
|
| 2381 |
},
|
| 2382 |
-
"capability_score": 98.
|
| 2383 |
-
"composite_score":
|
| 2384 |
"open_set_score": 100.0,
|
| 2385 |
-
"sealed_set_score":
|
| 2386 |
"calibration_details": {
|
| 2387 |
-
"null_auroc": 0.
|
| 2388 |
-
"null_ece": 0.
|
| 2389 |
-
"null_brier": 0.
|
| 2390 |
},
|
| 2391 |
"latency": {
|
| 2392 |
-
"mean_ms":
|
| 2393 |
-
"p50_ms":
|
| 2394 |
-
"p95_ms":
|
| 2395 |
-
"p99_ms":
|
| 2396 |
},
|
| 2397 |
"by_request_type": {
|
| 2398 |
"span_open": {
|
|
@@ -2401,11 +2672,11 @@
|
|
| 2401 |
},
|
| 2402 |
"math": {
|
| 2403 |
"score": 100.0,
|
| 2404 |
-
"n":
|
| 2405 |
},
|
| 2406 |
"choice": {
|
| 2407 |
"score": 100.0,
|
| 2408 |
-
"n":
|
| 2409 |
},
|
| 2410 |
"noul_open": {
|
| 2411 |
"score": 100.0,
|
|
@@ -2416,12 +2687,12 @@
|
|
| 2416 |
"n": 48
|
| 2417 |
},
|
| 2418 |
"span_sealed": {
|
| 2419 |
-
"score":
|
| 2420 |
-
"n":
|
| 2421 |
},
|
| 2422 |
"noul_sealed": {
|
| 2423 |
-
"score":
|
| 2424 |
-
"n":
|
| 2425 |
}
|
| 2426 |
},
|
| 2427 |
"by_tier_open": {
|
|
@@ -2440,12 +2711,16 @@
|
|
| 2440 |
},
|
| 2441 |
"by_tier_sealed": {
|
| 2442 |
"Standard": {
|
| 2443 |
-
"score":
|
| 2444 |
-
"n":
|
| 2445 |
},
|
| 2446 |
"Hard": {
|
| 2447 |
-
"score":
|
| 2448 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2449 |
}
|
| 2450 |
},
|
| 2451 |
"by_domain": {
|
|
@@ -2466,9 +2741,13 @@
|
|
| 2466 |
"n": 16
|
| 2467 |
},
|
| 2468 |
"squad_v2_open_qa": {
|
| 2469 |
-
"score":
|
| 2470 |
"n": 60
|
| 2471 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2472 |
"adversarial_stress": {
|
| 2473 |
"score": 100.0,
|
| 2474 |
"n": 8
|
|
@@ -2484,38 +2763,38 @@
|
|
| 2484 |
"open_weights": true,
|
| 2485 |
"params_m": 0.0,
|
| 2486 |
"footprint_mb": 0.0,
|
| 2487 |
-
"base_model": "Deterministic
|
| 2488 |
"runtime": "Pure Python / Node / Browser JS",
|
| 2489 |
"input_rate_per_1m_usd": 0.0008,
|
| 2490 |
"output_rate_per_1m_usd": 0.0,
|
| 2491 |
-
"n_observations":
|
| 2492 |
-
"input_tokens_total":
|
| 2493 |
"output_tokens_total": 0,
|
| 2494 |
"usd_per_1k_decisions": 0.0001,
|
| 2495 |
"axes": {
|
| 2496 |
-
"intelligence":
|
| 2497 |
"intelligence_ci95": [
|
| 2498 |
-
|
| 2499 |
-
|
| 2500 |
],
|
| 2501 |
-
"calibration":
|
| 2502 |
-
"speed":
|
| 2503 |
"cost": 99.8
|
| 2504 |
},
|
| 2505 |
-
"capability_score":
|
| 2506 |
-
"composite_score": 95.
|
| 2507 |
"open_set_score": 100.0,
|
| 2508 |
-
"sealed_set_score":
|
| 2509 |
"calibration_details": {
|
| 2510 |
-
"null_auroc": 0.
|
| 2511 |
-
"null_ece": 0.
|
| 2512 |
-
"null_brier": 0.
|
| 2513 |
},
|
| 2514 |
"latency": {
|
| 2515 |
-
"mean_ms": 0.
|
| 2516 |
-
"p50_ms": 0.
|
| 2517 |
-
"p95_ms": 0.
|
| 2518 |
-
"p99_ms": 0.
|
| 2519 |
},
|
| 2520 |
"by_request_type": {
|
| 2521 |
"span_open": {
|
|
@@ -2524,11 +2803,11 @@
|
|
| 2524 |
},
|
| 2525 |
"math": {
|
| 2526 |
"score": 100.0,
|
| 2527 |
-
"n":
|
| 2528 |
},
|
| 2529 |
"choice": {
|
| 2530 |
"score": 100.0,
|
| 2531 |
-
"n":
|
| 2532 |
},
|
| 2533 |
"noul_open": {
|
| 2534 |
"score": 100.0,
|
|
@@ -2539,12 +2818,12 @@
|
|
| 2539 |
"n": 48
|
| 2540 |
},
|
| 2541 |
"span_sealed": {
|
| 2542 |
-
"score":
|
| 2543 |
-
"n":
|
| 2544 |
},
|
| 2545 |
"noul_sealed": {
|
| 2546 |
-
"score":
|
| 2547 |
-
"n":
|
| 2548 |
}
|
| 2549 |
},
|
| 2550 |
"by_tier_open": {
|
|
@@ -2563,12 +2842,16 @@
|
|
| 2563 |
},
|
| 2564 |
"by_tier_sealed": {
|
| 2565 |
"Standard": {
|
| 2566 |
-
"score":
|
| 2567 |
-
"n":
|
| 2568 |
},
|
| 2569 |
"Hard": {
|
| 2570 |
-
"score":
|
| 2571 |
-
"n":
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2572 |
}
|
| 2573 |
},
|
| 2574 |
"by_domain": {
|
|
@@ -2592,6 +2875,10 @@
|
|
| 2592 |
"score": 23.3,
|
| 2593 |
"n": 60
|
| 2594 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2595 |
"adversarial_stress": {
|
| 2596 |
"score": 100.0,
|
| 2597 |
"n": 8
|
|
|
|
| 1 |
{
|
| 2 |
"benchmark_name": "NullspanBench (JevBench-Compatible Decision & Extraction Leaderboard)",
|
| 3 |
+
"benchmark_version": "3.2.0-jevbench",
|
| 4 |
+
"draw_id": "S+P (draw v3.2-nullspan-20261030)",
|
| 5 |
+
"timestamp": "2026-10-30T10:18:35Z",
|
| 6 |
+
"n_total_observations": 500,
|
| 7 |
"n_open_observations": 336,
|
| 8 |
+
"n_sealed_observations": 164,
|
| 9 |
"environment": {
|
| 10 |
"os": "Darwin 25.5.0 (arm64)",
|
| 11 |
"python": "3.14.8",
|
| 12 |
"pytorch": "2.12.1",
|
| 13 |
"onnxruntime": "1.31.0",
|
| 14 |
+
"accelerator": "NVIDIA Tesla V100-SXM2-16GB (GCP Train) + Apple Silicon MPS/CPU (Local Eval)"
|
| 15 |
},
|
| 16 |
"systems": [
|
| 17 |
{
|
|
|
|
| 20 |
"author": "gnumanth",
|
| 21 |
"system_type": "Hybrid System One Router",
|
| 22 |
"open_weights": true,
|
| 23 |
+
"params_m": 344.2,
|
| 24 |
+
"footprint_mb": 330.1,
|
| 25 |
"base_model": "Zero-Dep Router + gnumanth/nullspan Int8",
|
| 26 |
"runtime": "Tiered JS/Python + ONNX Runtime",
|
| 27 |
+
"input_rate_per_1m_usd": 0.002,
|
| 28 |
"output_rate_per_1m_usd": 0.0,
|
| 29 |
+
"n_observations": 500,
|
| 30 |
+
"input_tokens_total": 69943,
|
| 31 |
"output_tokens_total": 0,
|
| 32 |
"usd_per_1k_decisions": 0.0003,
|
| 33 |
"axes": {
|
| 34 |
+
"intelligence": 97.8,
|
| 35 |
"intelligence_ci95": [
|
| 36 |
+
96.6,
|
| 37 |
+
99.0
|
| 38 |
],
|
| 39 |
+
"calibration": 98.8,
|
| 40 |
+
"speed": 98.6,
|
| 41 |
"cost": 93.8
|
| 42 |
},
|
| 43 |
+
"capability_score": 98.3,
|
| 44 |
+
"composite_score": 97.2,
|
| 45 |
"open_set_score": 100.0,
|
| 46 |
+
"sealed_set_score": 93.3,
|
| 47 |
"calibration_details": {
|
| 48 |
+
"null_auroc": 0.9977,
|
| 49 |
+
"null_ece": 0.011,
|
| 50 |
+
"null_brier": 0.01306
|
| 51 |
},
|
| 52 |
"latency": {
|
| 53 |
+
"mean_ms": 14.3016,
|
| 54 |
+
"p50_ms": 0.013,
|
| 55 |
+
"p95_ms": 108.7561,
|
| 56 |
+
"p99_ms": 178.0133
|
| 57 |
},
|
| 58 |
"by_request_type": {
|
| 59 |
"span_open": {
|
|
|
|
| 62 |
},
|
| 63 |
"math": {
|
| 64 |
"score": 100.0,
|
| 65 |
+
"n": 51
|
| 66 |
},
|
| 67 |
"choice": {
|
| 68 |
"score": 100.0,
|
| 69 |
+
"n": 41
|
| 70 |
},
|
| 71 |
"noul_open": {
|
| 72 |
"score": 100.0,
|
|
|
|
| 77 |
"n": 48
|
| 78 |
},
|
| 79 |
"span_sealed": {
|
| 80 |
+
"score": 95.4,
|
| 81 |
+
"n": 108
|
| 82 |
},
|
| 83 |
"noul_sealed": {
|
| 84 |
+
"score": 86.4,
|
| 85 |
+
"n": 44
|
| 86 |
}
|
| 87 |
},
|
| 88 |
"by_tier_open": {
|
|
|
|
| 101 |
},
|
| 102 |
"by_tier_sealed": {
|
| 103 |
"Standard": {
|
| 104 |
+
"score": 95.6,
|
| 105 |
+
"n": 113
|
| 106 |
},
|
| 107 |
"Hard": {
|
| 108 |
+
"score": 88.0,
|
| 109 |
+
"n": 50
|
| 110 |
+
},
|
| 111 |
+
"Judge": {
|
| 112 |
+
"score": 100.0,
|
| 113 |
+
"n": 1
|
| 114 |
}
|
| 115 |
},
|
| 116 |
"by_domain": {
|
|
|
|
| 131 |
"n": 16
|
| 132 |
},
|
| 133 |
"squad_v2_open_qa": {
|
| 134 |
+
"score": 81.7,
|
| 135 |
"n": 60
|
| 136 |
},
|
| 137 |
+
"contract_extraction": {
|
| 138 |
+
"score": 100.0,
|
| 139 |
+
"n": 96
|
| 140 |
+
},
|
| 141 |
"adversarial_stress": {
|
| 142 |
"score": 100.0,
|
| 143 |
"n": 8
|
|
|
|
| 153 |
"open_weights": true,
|
| 154 |
"params_m": 0.0,
|
| 155 |
"footprint_mb": 0.0,
|
| 156 |
+
"base_model": "Deterministic Clause + Lexical Pointer",
|
| 157 |
"runtime": "Pure Python / Node / Browser JS",
|
| 158 |
"input_rate_per_1m_usd": 0.0008,
|
| 159 |
"output_rate_per_1m_usd": 0.0,
|
| 160 |
+
"n_observations": 500,
|
| 161 |
+
"input_tokens_total": 69943,
|
| 162 |
"output_tokens_total": 0,
|
| 163 |
"usd_per_1k_decisions": 0.0001,
|
| 164 |
"axes": {
|
| 165 |
+
"intelligence": 90.8,
|
| 166 |
"intelligence_ci95": [
|
| 167 |
+
88.2,
|
| 168 |
+
93.4
|
| 169 |
],
|
| 170 |
+
"calibration": 95.0,
|
| 171 |
+
"speed": 98.7,
|
| 172 |
"cost": 99.8
|
| 173 |
},
|
| 174 |
+
"capability_score": 92.9,
|
| 175 |
+
"composite_score": 95.9,
|
| 176 |
"open_set_score": 100.0,
|
| 177 |
+
"sealed_set_score": 72.0,
|
| 178 |
"calibration_details": {
|
| 179 |
+
"null_auroc": 0.98239,
|
| 180 |
+
"null_ece": 0.04866,
|
| 181 |
+
"null_brier": 0.05142
|
| 182 |
},
|
| 183 |
"latency": {
|
| 184 |
+
"mean_ms": 0.0632,
|
| 185 |
+
"p50_ms": 0.0127,
|
| 186 |
+
"p95_ms": 0.2868,
|
| 187 |
+
"p99_ms": 0.3604
|
| 188 |
},
|
| 189 |
"by_request_type": {
|
| 190 |
"span_open": {
|
|
|
|
| 193 |
},
|
| 194 |
"math": {
|
| 195 |
"score": 100.0,
|
| 196 |
+
"n": 51
|
| 197 |
},
|
| 198 |
"choice": {
|
| 199 |
"score": 100.0,
|
| 200 |
+
"n": 41
|
| 201 |
},
|
| 202 |
"noul_open": {
|
| 203 |
"score": 100.0,
|
|
|
|
| 208 |
"n": 48
|
| 209 |
},
|
| 210 |
"span_sealed": {
|
| 211 |
+
"score": 78.7,
|
| 212 |
+
"n": 108
|
| 213 |
},
|
| 214 |
"noul_sealed": {
|
| 215 |
+
"score": 47.7,
|
| 216 |
+
"n": 44
|
| 217 |
}
|
| 218 |
},
|
| 219 |
"by_tier_open": {
|
|
|
|
| 232 |
},
|
| 233 |
"by_tier_sealed": {
|
| 234 |
"Standard": {
|
| 235 |
+
"score": 79.6,
|
| 236 |
+
"n": 113
|
| 237 |
},
|
| 238 |
"Hard": {
|
| 239 |
+
"score": 54.0,
|
| 240 |
+
"n": 50
|
| 241 |
+
},
|
| 242 |
+
"Judge": {
|
| 243 |
+
"score": 100.0,
|
| 244 |
+
"n": 1
|
| 245 |
}
|
| 246 |
},
|
| 247 |
"by_domain": {
|
|
|
|
| 265 |
"score": 23.3,
|
| 266 |
"n": 60
|
| 267 |
},
|
| 268 |
+
"contract_extraction": {
|
| 269 |
+
"score": 100.0,
|
| 270 |
+
"n": 96
|
| 271 |
+
},
|
| 272 |
"adversarial_stress": {
|
| 273 |
"score": 100.0,
|
| 274 |
"n": 8
|
|
|
|
| 277 |
"official_rank": 2
|
| 278 |
},
|
| 279 |
{
|
| 280 |
+
"id": "nullspan-fp32",
|
| 281 |
+
"name": "nullspan-344M (PyTorch FP32)",
|
| 282 |
"author": "gnumanth",
|
| 283 |
+
"system_type": "System One Encoder (5-Head, 512-ctx)",
|
| 284 |
"open_weights": true,
|
| 285 |
+
"params_m": 344.2,
|
| 286 |
+
"footprint_mb": 1313.1,
|
| 287 |
+
"base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter",
|
| 288 |
+
"runtime": "PyTorch MPS/CUDA/CPU FP32",
|
| 289 |
+
"input_rate_per_1m_usd": 0.012,
|
| 290 |
"output_rate_per_1m_usd": 0.0,
|
| 291 |
+
"n_observations": 500,
|
| 292 |
+
"input_tokens_total": 69943,
|
| 293 |
"output_tokens_total": 0,
|
| 294 |
+
"usd_per_1k_decisions": 0.0017,
|
| 295 |
"axes": {
|
| 296 |
+
"intelligence": 98.0,
|
| 297 |
"intelligence_ci95": [
|
| 298 |
+
96.8,
|
| 299 |
+
99.2
|
| 300 |
],
|
| 301 |
+
"calibration": 98.8,
|
| 302 |
+
"speed": 57.2,
|
| 303 |
+
"cost": 76.8
|
| 304 |
},
|
| 305 |
+
"capability_score": 98.4,
|
| 306 |
+
"composite_score": 78.7,
|
| 307 |
"open_set_score": 100.0,
|
| 308 |
+
"sealed_set_score": 93.9,
|
| 309 |
"calibration_details": {
|
| 310 |
+
"null_auroc": 0.99829,
|
| 311 |
+
"null_ece": 0.01215,
|
| 312 |
+
"null_brier": 0.01245
|
| 313 |
},
|
| 314 |
"latency": {
|
| 315 |
+
"mean_ms": 47.0471,
|
| 316 |
+
"p50_ms": 32.3672,
|
| 317 |
+
"p95_ms": 91.8385,
|
| 318 |
+
"p99_ms": 108.335
|
| 319 |
},
|
| 320 |
"by_request_type": {
|
| 321 |
"span_open": {
|
|
|
|
| 324 |
},
|
| 325 |
"math": {
|
| 326 |
"score": 100.0,
|
| 327 |
+
"n": 51
|
| 328 |
},
|
| 329 |
"choice": {
|
| 330 |
"score": 100.0,
|
| 331 |
+
"n": 41
|
| 332 |
},
|
| 333 |
"noul_open": {
|
| 334 |
"score": 100.0,
|
|
|
|
| 339 |
"n": 48
|
| 340 |
},
|
| 341 |
"span_sealed": {
|
| 342 |
+
"score": 96.3,
|
| 343 |
+
"n": 108
|
| 344 |
},
|
| 345 |
"noul_sealed": {
|
| 346 |
+
"score": 86.4,
|
| 347 |
+
"n": 44
|
| 348 |
}
|
| 349 |
},
|
| 350 |
"by_tier_open": {
|
|
|
|
| 363 |
},
|
| 364 |
"by_tier_sealed": {
|
| 365 |
"Standard": {
|
| 366 |
+
"score": 96.5,
|
| 367 |
+
"n": 113
|
| 368 |
},
|
| 369 |
"Hard": {
|
| 370 |
+
"score": 88.0,
|
| 371 |
+
"n": 50
|
| 372 |
+
},
|
| 373 |
+
"Judge": {
|
| 374 |
"score": 100.0,
|
| 375 |
+
"n": 1
|
| 376 |
}
|
| 377 |
},
|
| 378 |
"by_domain": {
|
|
|
|
| 393 |
"n": 16
|
| 394 |
},
|
| 395 |
"squad_v2_open_qa": {
|
| 396 |
+
"score": 83.3,
|
| 397 |
"n": 60
|
| 398 |
},
|
| 399 |
+
"contract_extraction": {
|
| 400 |
+
"score": 100.0,
|
| 401 |
+
"n": 96
|
| 402 |
+
},
|
| 403 |
"adversarial_stress": {
|
| 404 |
"score": 100.0,
|
| 405 |
"n": 8
|
|
|
|
| 408 |
"official_rank": 3
|
| 409 |
},
|
| 410 |
{
|
| 411 |
+
"id": "nullspan-onnx-int8",
|
| 412 |
+
"name": "nullspan-344M-Int8 (ONNX WASM/CPU)",
|
| 413 |
"author": "gnumanth",
|
| 414 |
+
"system_type": "Quantized ONNX WASM/CPU (512-ctx)",
|
| 415 |
"open_weights": true,
|
| 416 |
+
"params_m": 344.2,
|
| 417 |
+
"footprint_mb": 330.1,
|
| 418 |
"base_model": "gnumanth/nullspan (Int8 Dynamic)",
|
| 419 |
"runtime": "ONNX Runtime Int8 (Browser WASM / Edge)",
|
| 420 |
+
"input_rate_per_1m_usd": 0.006,
|
| 421 |
"output_rate_per_1m_usd": 0.0,
|
| 422 |
+
"n_observations": 500,
|
| 423 |
+
"input_tokens_total": 69943,
|
| 424 |
"output_tokens_total": 0,
|
| 425 |
"usd_per_1k_decisions": 0.0008,
|
| 426 |
"axes": {
|
| 427 |
+
"intelligence": 97.8,
|
| 428 |
"intelligence_ci95": [
|
| 429 |
+
96.6,
|
| 430 |
+
99.0
|
| 431 |
],
|
| 432 |
+
"calibration": 98.9,
|
| 433 |
+
"speed": 53.2,
|
| 434 |
"cost": 84.2
|
| 435 |
},
|
| 436 |
+
"capability_score": 98.3,
|
| 437 |
+
"composite_score": 78.4,
|
| 438 |
"open_set_score": 100.0,
|
| 439 |
+
"sealed_set_score": 93.3,
|
| 440 |
"calibration_details": {
|
| 441 |
+
"null_auroc": 0.99737,
|
| 442 |
+
"null_ece": 0.0095,
|
| 443 |
+
"null_brier": 0.01314
|
| 444 |
},
|
| 445 |
"latency": {
|
| 446 |
+
"mean_ms": 107.1327,
|
| 447 |
+
"p50_ms": 68.4825,
|
| 448 |
+
"p95_ms": 262.6889,
|
| 449 |
+
"p99_ms": 311.1327
|
| 450 |
},
|
| 451 |
"by_request_type": {
|
| 452 |
"span_open": {
|
|
|
|
| 455 |
},
|
| 456 |
"math": {
|
| 457 |
"score": 100.0,
|
| 458 |
+
"n": 51
|
| 459 |
},
|
| 460 |
"choice": {
|
| 461 |
"score": 100.0,
|
| 462 |
+
"n": 41
|
| 463 |
},
|
| 464 |
"noul_open": {
|
| 465 |
"score": 100.0,
|
|
|
|
| 470 |
"n": 48
|
| 471 |
},
|
| 472 |
"span_sealed": {
|
| 473 |
+
"score": 95.4,
|
| 474 |
+
"n": 108
|
| 475 |
},
|
| 476 |
"noul_sealed": {
|
| 477 |
+
"score": 86.4,
|
| 478 |
+
"n": 44
|
| 479 |
}
|
| 480 |
},
|
| 481 |
"by_tier_open": {
|
|
|
|
| 494 |
},
|
| 495 |
"by_tier_sealed": {
|
| 496 |
"Standard": {
|
| 497 |
+
"score": 95.6,
|
| 498 |
+
"n": 113
|
| 499 |
},
|
| 500 |
"Hard": {
|
| 501 |
+
"score": 88.0,
|
| 502 |
+
"n": 50
|
| 503 |
+
},
|
| 504 |
+
"Judge": {
|
| 505 |
+
"score": 100.0,
|
| 506 |
+
"n": 1
|
| 507 |
}
|
| 508 |
},
|
| 509 |
"by_domain": {
|
|
|
|
| 524 |
"n": 16
|
| 525 |
},
|
| 526 |
"squad_v2_open_qa": {
|
| 527 |
+
"score": 81.7,
|
| 528 |
"n": 60
|
| 529 |
},
|
| 530 |
+
"contract_extraction": {
|
| 531 |
+
"score": 100.0,
|
| 532 |
+
"n": 96
|
| 533 |
+
},
|
| 534 |
"adversarial_stress": {
|
| 535 |
"score": 100.0,
|
| 536 |
"n": 8
|
|
|
|
| 539 |
"official_rank": 4
|
| 540 |
},
|
| 541 |
{
|
| 542 |
+
"id": "nullspan-onnx-fp32",
|
| 543 |
+
"name": "nullspan-344M (ONNX FP32)",
|
| 544 |
"author": "gnumanth",
|
| 545 |
+
"system_type": "System One Encoder (ONNX, 512-ctx)",
|
| 546 |
"open_weights": true,
|
| 547 |
+
"params_m": 344.2,
|
| 548 |
+
"footprint_mb": 1313.6,
|
| 549 |
"base_model": "gnumanth/nullspan (ONNX Opset 17)",
|
| 550 |
"runtime": "ONNX Runtime FP32",
|
| 551 |
+
"input_rate_per_1m_usd": 0.011,
|
| 552 |
"output_rate_per_1m_usd": 0.0,
|
| 553 |
+
"n_observations": 500,
|
| 554 |
+
"input_tokens_total": 69943,
|
| 555 |
"output_tokens_total": 0,
|
| 556 |
+
"usd_per_1k_decisions": 0.0015,
|
| 557 |
"axes": {
|
| 558 |
+
"intelligence": 98.0,
|
| 559 |
"intelligence_ci95": [
|
| 560 |
+
96.8,
|
| 561 |
+
99.2
|
| 562 |
],
|
| 563 |
+
"calibration": 98.8,
|
| 564 |
+
"speed": 50.5,
|
| 565 |
+
"cost": 78.0
|
| 566 |
},
|
| 567 |
+
"capability_score": 98.4,
|
| 568 |
+
"composite_score": 75.5,
|
| 569 |
"open_set_score": 100.0,
|
| 570 |
+
"sealed_set_score": 93.9,
|
| 571 |
"calibration_details": {
|
| 572 |
+
"null_auroc": 0.99829,
|
| 573 |
+
"null_ece": 0.01215,
|
| 574 |
+
"null_brier": 0.01245
|
| 575 |
},
|
| 576 |
"latency": {
|
| 577 |
+
"mean_ms": 134.8996,
|
| 578 |
+
"p50_ms": 113.1368,
|
| 579 |
+
"p95_ms": 225.285,
|
| 580 |
+
"p99_ms": 513.2005
|
| 581 |
},
|
| 582 |
"by_request_type": {
|
| 583 |
"span_open": {
|
|
|
|
| 586 |
},
|
| 587 |
"math": {
|
| 588 |
"score": 100.0,
|
| 589 |
+
"n": 51
|
| 590 |
},
|
| 591 |
"choice": {
|
| 592 |
"score": 100.0,
|
| 593 |
+
"n": 41
|
| 594 |
},
|
| 595 |
"noul_open": {
|
| 596 |
"score": 100.0,
|
|
|
|
| 601 |
"n": 48
|
| 602 |
},
|
| 603 |
"span_sealed": {
|
| 604 |
+
"score": 96.3,
|
| 605 |
+
"n": 108
|
| 606 |
},
|
| 607 |
"noul_sealed": {
|
| 608 |
+
"score": 86.4,
|
| 609 |
+
"n": 44
|
| 610 |
}
|
| 611 |
},
|
| 612 |
"by_tier_open": {
|
|
|
|
| 625 |
},
|
| 626 |
"by_tier_sealed": {
|
| 627 |
"Standard": {
|
| 628 |
+
"score": 96.5,
|
| 629 |
+
"n": 113
|
| 630 |
},
|
| 631 |
"Hard": {
|
| 632 |
+
"score": 88.0,
|
| 633 |
+
"n": 50
|
| 634 |
+
},
|
| 635 |
+
"Judge": {
|
| 636 |
"score": 100.0,
|
| 637 |
+
"n": 1
|
| 638 |
}
|
| 639 |
},
|
| 640 |
"by_domain": {
|
|
|
|
| 655 |
"n": 16
|
| 656 |
},
|
| 657 |
"squad_v2_open_qa": {
|
| 658 |
+
"score": 83.3,
|
| 659 |
"n": 60
|
| 660 |
},
|
| 661 |
+
"contract_extraction": {
|
| 662 |
+
"score": 100.0,
|
| 663 |
+
"n": 96
|
| 664 |
+
},
|
| 665 |
"adversarial_stress": {
|
| 666 |
"score": 100.0,
|
| 667 |
"n": 8
|
|
|
|
| 785 |
"score": 62.6,
|
| 786 |
"n": 300
|
| 787 |
},
|
| 788 |
+
"contract_extraction": {
|
| 789 |
+
"score": 58.4,
|
| 790 |
+
"n": 300
|
| 791 |
+
},
|
| 792 |
"adversarial_stress": {
|
| 793 |
"score": 54.3,
|
| 794 |
"n": 300
|
|
|
|
| 796 |
},
|
| 797 |
"official_rank": 6
|
| 798 |
},
|
| 799 |
+
{
|
| 800 |
+
"id": "cloudflare-clairic-clef-live",
|
| 801 |
+
"name": "@cf/cloudflare/clairic + clef (Live Workers AI)",
|
| 802 |
+
"author": "Cloudflare",
|
| 803 |
+
"system_type": "System One Split API (Clairic + Clef 27B)",
|
| 804 |
+
"open_weights": false,
|
| 805 |
+
"params_m": 27000.0,
|
| 806 |
+
"footprint_mb": null,
|
| 807 |
+
"base_model": "@cf/cloudflare/clairic + @cf/cloudflare/clef (180 RPCs / 246 Passes)",
|
| 808 |
+
"runtime": "Cloudflare Workers AI Edge RPC (p50 315.3 ms/doc)",
|
| 809 |
+
"input_rate_per_1m_usd": 0.025,
|
| 810 |
+
"output_rate_per_1m_usd": 0.0,
|
| 811 |
+
"n_observations": 500,
|
| 812 |
+
"input_tokens_total": 81206,
|
| 813 |
+
"output_tokens_total": 0,
|
| 814 |
+
"usd_per_1k_decisions": 0.0041,
|
| 815 |
+
"axes": {
|
| 816 |
+
"intelligence": 91.6,
|
| 817 |
+
"intelligence_ci95": [
|
| 818 |
+
89.2,
|
| 819 |
+
93.9
|
| 820 |
+
],
|
| 821 |
+
"calibration": 94.9,
|
| 822 |
+
"speed": 51.7,
|
| 823 |
+
"cost": 68.2
|
| 824 |
+
},
|
| 825 |
+
"capability_score": 93.2,
|
| 826 |
+
"composite_score": 72.1,
|
| 827 |
+
"open_set_score": 89.7,
|
| 828 |
+
"sealed_set_score": 95.4,
|
| 829 |
+
"calibration_details": {
|
| 830 |
+
"null_auroc": 0.98959,
|
| 831 |
+
"null_ece": 0.05235,
|
| 832 |
+
"null_brier": 0.04728
|
| 833 |
+
},
|
| 834 |
+
"latency": {
|
| 835 |
+
"mean_ms": 128.789,
|
| 836 |
+
"p50_ms": 91.323,
|
| 837 |
+
"p95_ms": 314.023,
|
| 838 |
+
"p99_ms": 403.1387
|
| 839 |
+
},
|
| 840 |
+
"by_request_type": {
|
| 841 |
+
"span_open": {
|
| 842 |
+
"score": 83.8,
|
| 843 |
+
"n": 181
|
| 844 |
+
},
|
| 845 |
+
"math": {
|
| 846 |
+
"score": 100.0,
|
| 847 |
+
"n": 51
|
| 848 |
+
},
|
| 849 |
+
"choice": {
|
| 850 |
+
"score": 100.0,
|
| 851 |
+
"n": 41
|
| 852 |
+
},
|
| 853 |
+
"noul_open": {
|
| 854 |
+
"score": 100.0,
|
| 855 |
+
"n": 27
|
| 856 |
+
},
|
| 857 |
+
"score": {
|
| 858 |
+
"score": 89.0,
|
| 859 |
+
"n": 48
|
| 860 |
+
},
|
| 861 |
+
"span_sealed": {
|
| 862 |
+
"score": 95.8,
|
| 863 |
+
"n": 108
|
| 864 |
+
},
|
| 865 |
+
"noul_sealed": {
|
| 866 |
+
"score": 93.2,
|
| 867 |
+
"n": 44
|
| 868 |
+
}
|
| 869 |
+
},
|
| 870 |
+
"by_tier_open": {
|
| 871 |
+
"Easy": {
|
| 872 |
+
"score": 85.8,
|
| 873 |
+
"n": 176
|
| 874 |
+
},
|
| 875 |
+
"Judge": {
|
| 876 |
+
"score": 94.5,
|
| 877 |
+
"n": 96
|
| 878 |
+
},
|
| 879 |
+
"Standard": {
|
| 880 |
+
"score": 93.4,
|
| 881 |
+
"n": 64
|
| 882 |
+
}
|
| 883 |
+
},
|
| 884 |
+
"by_tier_sealed": {
|
| 885 |
+
"Standard": {
|
| 886 |
+
"score": 96.9,
|
| 887 |
+
"n": 113
|
| 888 |
+
},
|
| 889 |
+
"Hard": {
|
| 890 |
+
"score": 92.0,
|
| 891 |
+
"n": 50
|
| 892 |
+
},
|
| 893 |
+
"Judge": {
|
| 894 |
+
"score": 100.0,
|
| 895 |
+
"n": 1
|
| 896 |
+
}
|
| 897 |
+
},
|
| 898 |
+
"by_domain": {
|
| 899 |
+
"incident_report": {
|
| 900 |
+
"score": 94.4,
|
| 901 |
+
"n": 160
|
| 902 |
+
},
|
| 903 |
+
"compute_ledger": {
|
| 904 |
+
"score": 82.6,
|
| 905 |
+
"n": 144
|
| 906 |
+
},
|
| 907 |
+
"cluster_throughput_comps": {
|
| 908 |
+
"score": 96.2,
|
| 909 |
+
"n": 16
|
| 910 |
+
},
|
| 911 |
+
"edge_traffic_forecast": {
|
| 912 |
+
"score": 100.0,
|
| 913 |
+
"n": 16
|
| 914 |
+
},
|
| 915 |
+
"squad_v2_open_qa": {
|
| 916 |
+
"score": 92.5,
|
| 917 |
+
"n": 60
|
| 918 |
+
},
|
| 919 |
+
"contract_extraction": {
|
| 920 |
+
"score": 97.9,
|
| 921 |
+
"n": 96
|
| 922 |
+
},
|
| 923 |
+
"adversarial_stress": {
|
| 924 |
+
"score": 87.5,
|
| 925 |
+
"n": 8
|
| 926 |
+
}
|
| 927 |
+
},
|
| 928 |
+
"official_rank": 7
|
| 929 |
+
},
|
| 930 |
{
|
| 931 |
"id": "basal-omni-1.0-ultra",
|
| 932 |
"name": "basal-omni-1.0-ultra (JevBench #1 Cap)",
|
|
|
|
| 1033 |
},
|
| 1034 |
"cluster_throughput_comps": {
|
| 1035 |
"score": 69.1,
|
| 1036 |
+
"n": 300
|
| 1037 |
},
|
| 1038 |
"edge_traffic_forecast": {
|
| 1039 |
"score": 66.9,
|
|
|
|
| 1043 |
"score": 76.9,
|
| 1044 |
"n": 300
|
| 1045 |
},
|
| 1046 |
+
"contract_extraction": {
|
| 1047 |
+
"score": 72.8,
|
| 1048 |
+
"n": 300
|
| 1049 |
+
},
|
| 1050 |
"adversarial_stress": {
|
| 1051 |
"score": 74.0,
|
| 1052 |
"n": 300
|
| 1053 |
}
|
| 1054 |
},
|
| 1055 |
+
"official_rank": 8
|
| 1056 |
},
|
| 1057 |
{
|
| 1058 |
"id": "jev-1.13.0-ref",
|
|
|
|
| 1170 |
"score": 61.6,
|
| 1171 |
"n": 300
|
| 1172 |
},
|
| 1173 |
+
"contract_extraction": {
|
| 1174 |
+
"score": 62.4,
|
| 1175 |
+
"n": 300
|
| 1176 |
+
},
|
| 1177 |
"adversarial_stress": {
|
| 1178 |
"score": 58.9,
|
| 1179 |
"n": 300
|
| 1180 |
}
|
| 1181 |
},
|
| 1182 |
+
"official_rank": 9
|
| 1183 |
},
|
| 1184 |
{
|
| 1185 |
+
"id": "gpt-5.4-nano-json",
|
| 1186 |
+
"name": "gpt-5.4-nano (JSON Decoder \u00b7 API)",
|
| 1187 |
+
"author": "OpenAI (Ref)",
|
| 1188 |
+
"system_type": "Closed API \u00b7 LLM Decoder",
|
| 1189 |
"open_weights": false,
|
| 1190 |
+
"params_m": null,
|
| 1191 |
"footprint_mb": null,
|
| 1192 |
+
"base_model": "Autoregressive JSON Schema Decoder",
|
| 1193 |
+
"runtime": "Hosted Cloud API ( ~38 output tokens/field )",
|
| 1194 |
+
"input_rate_per_1m_usd": 0.15,
|
| 1195 |
+
"output_rate_per_1m_usd": 0.6,
|
| 1196 |
+
"n_observations": 500,
|
| 1197 |
+
"input_tokens_total": 69943,
|
| 1198 |
+
"output_tokens_total": 19000,
|
| 1199 |
+
"usd_per_1k_decisions": 0.0438,
|
| 1200 |
"axes": {
|
| 1201 |
+
"intelligence": 95.2,
|
| 1202 |
"intelligence_ci95": [
|
| 1203 |
+
93.4,
|
| 1204 |
+
97.0
|
| 1205 |
],
|
| 1206 |
+
"calibration": 89.0,
|
| 1207 |
+
"speed": 43.6,
|
| 1208 |
+
"cost": 45.1
|
| 1209 |
},
|
| 1210 |
+
"capability_score": 92.1,
|
| 1211 |
+
"composite_score": 59.8,
|
| 1212 |
+
"open_set_score": 99.7,
|
| 1213 |
+
"sealed_set_score": 86.0,
|
| 1214 |
"calibration_details": {
|
| 1215 |
+
"null_auroc": 0.95791,
|
| 1216 |
+
"null_ece": 0.15112,
|
| 1217 |
+
"null_brier": 0.03403
|
| 1218 |
},
|
| 1219 |
"latency": {
|
| 1220 |
+
"mean_ms": 420.4635,
|
| 1221 |
+
"p50_ms": 417.9908,
|
| 1222 |
+
"p95_ms": 515.593,
|
| 1223 |
+
"p99_ms": 524.2688
|
| 1224 |
},
|
| 1225 |
"by_request_type": {
|
| 1226 |
"span_open": {
|
| 1227 |
+
"score": 100.0,
|
| 1228 |
"n": 181
|
| 1229 |
},
|
| 1230 |
"math": {
|
| 1231 |
"score": 100.0,
|
| 1232 |
+
"n": 51
|
| 1233 |
},
|
| 1234 |
"choice": {
|
| 1235 |
"score": 100.0,
|
| 1236 |
+
"n": 41
|
| 1237 |
},
|
| 1238 |
"noul_open": {
|
| 1239 |
+
"score": 96.3,
|
| 1240 |
"n": 27
|
| 1241 |
},
|
| 1242 |
"score": {
|
| 1243 |
+
"score": 100.0,
|
| 1244 |
"n": 48
|
| 1245 |
},
|
| 1246 |
"span_sealed": {
|
| 1247 |
+
"score": 91.7,
|
| 1248 |
+
"n": 108
|
| 1249 |
},
|
| 1250 |
"noul_sealed": {
|
| 1251 |
+
"score": 68.2,
|
| 1252 |
+
"n": 44
|
| 1253 |
}
|
| 1254 |
},
|
| 1255 |
"by_tier_open": {
|
| 1256 |
"Easy": {
|
| 1257 |
+
"score": 99.4,
|
| 1258 |
"n": 176
|
| 1259 |
},
|
| 1260 |
"Judge": {
|
| 1261 |
+
"score": 100.0,
|
| 1262 |
"n": 96
|
| 1263 |
},
|
| 1264 |
"Standard": {
|
| 1265 |
+
"score": 100.0,
|
| 1266 |
"n": 64
|
| 1267 |
}
|
| 1268 |
},
|
| 1269 |
"by_tier_sealed": {
|
| 1270 |
"Standard": {
|
| 1271 |
+
"score": 92.0,
|
| 1272 |
+
"n": 113
|
| 1273 |
},
|
| 1274 |
"Hard": {
|
| 1275 |
+
"score": 72.0,
|
| 1276 |
+
"n": 50
|
| 1277 |
+
},
|
| 1278 |
+
"Judge": {
|
| 1279 |
+
"score": 100.0,
|
| 1280 |
+
"n": 1
|
| 1281 |
}
|
| 1282 |
},
|
| 1283 |
"by_domain": {
|
| 1284 |
"incident_report": {
|
| 1285 |
+
"score": 99.4,
|
| 1286 |
"n": 160
|
| 1287 |
},
|
| 1288 |
"compute_ledger": {
|
| 1289 |
+
"score": 100.0,
|
| 1290 |
"n": 144
|
| 1291 |
},
|
| 1292 |
"cluster_throughput_comps": {
|
| 1293 |
+
"score": 100.0,
|
| 1294 |
"n": 16
|
| 1295 |
},
|
| 1296 |
"edge_traffic_forecast": {
|
|
|
|
| 1298 |
"n": 16
|
| 1299 |
},
|
| 1300 |
"squad_v2_open_qa": {
|
| 1301 |
+
"score": 76.7,
|
| 1302 |
"n": 60
|
| 1303 |
},
|
| 1304 |
+
"contract_extraction": {
|
| 1305 |
+
"score": 91.7,
|
| 1306 |
+
"n": 96
|
| 1307 |
+
},
|
| 1308 |
"adversarial_stress": {
|
| 1309 |
"score": 87.5,
|
| 1310 |
"n": 8
|
| 1311 |
}
|
| 1312 |
},
|
| 1313 |
+
"official_rank": 10
|
| 1314 |
},
|
| 1315 |
{
|
| 1316 |
+
"id": "deberta-v3-base-squad2",
|
| 1317 |
+
"name": "deberta-v3-base-squad2 (184M SQuAD2 QA)",
|
| 1318 |
+
"author": "deepset",
|
| 1319 |
+
"system_type": "Open Weights \u00b7 Extractive QA Cross-Encoder",
|
| 1320 |
+
"open_weights": true,
|
| 1321 |
+
"params_m": 183.8,
|
| 1322 |
+
"footprint_mb": 701.3,
|
| 1323 |
+
"base_model": "deepset/deberta-v3-base-squad2",
|
| 1324 |
+
"runtime": "PyTorch MPS/CPU (512-ctx)",
|
| 1325 |
+
"input_rate_per_1m_usd": 0.032,
|
| 1326 |
+
"output_rate_per_1m_usd": 0.0,
|
| 1327 |
+
"n_observations": 500,
|
| 1328 |
+
"input_tokens_total": 57964,
|
| 1329 |
+
"output_tokens_total": 0,
|
| 1330 |
+
"usd_per_1k_decisions": 0.0037,
|
| 1331 |
"axes": {
|
| 1332 |
+
"intelligence": 48.1,
|
| 1333 |
"intelligence_ci95": [
|
| 1334 |
+
43.9,
|
| 1335 |
+
52.4
|
| 1336 |
],
|
| 1337 |
+
"calibration": 63.4,
|
| 1338 |
+
"speed": 55.7,
|
| 1339 |
+
"cost": 69.2
|
| 1340 |
+
},
|
| 1341 |
+
"capability_score": 55.8,
|
| 1342 |
+
"composite_score": 58.0,
|
| 1343 |
+
"open_set_score": 48.6,
|
| 1344 |
+
"sealed_set_score": 47.3,
|
| 1345 |
"calibration_details": {
|
| 1346 |
+
"null_auroc": 0.82087,
|
| 1347 |
+
"null_ece": 0.39493,
|
| 1348 |
+
"null_brier": 0.31346
|
| 1349 |
},
|
| 1350 |
"latency": {
|
| 1351 |
+
"mean_ms": 48.5912,
|
| 1352 |
+
"p50_ms": 42.4964,
|
| 1353 |
+
"p95_ms": 111.7974,
|
| 1354 |
+
"p99_ms": 162.0547
|
| 1355 |
},
|
| 1356 |
"by_request_type": {
|
| 1357 |
"span_open": {
|
| 1358 |
+
"score": 56.4,
|
| 1359 |
"n": 181
|
| 1360 |
},
|
| 1361 |
"math": {
|
| 1362 |
+
"score": 4.9,
|
| 1363 |
+
"n": 51
|
| 1364 |
},
|
| 1365 |
"choice": {
|
| 1366 |
+
"score": 65.9,
|
| 1367 |
+
"n": 41
|
| 1368 |
},
|
| 1369 |
"noul_open": {
|
| 1370 |
+
"score": 100.0,
|
| 1371 |
"n": 27
|
| 1372 |
},
|
| 1373 |
"score": {
|
| 1374 |
+
"score": 15.1,
|
| 1375 |
"n": 48
|
| 1376 |
},
|
| 1377 |
"span_sealed": {
|
| 1378 |
+
"score": 35.2,
|
| 1379 |
+
"n": 108
|
| 1380 |
},
|
| 1381 |
"noul_sealed": {
|
| 1382 |
+
"score": 84.1,
|
| 1383 |
+
"n": 44
|
| 1384 |
}
|
| 1385 |
},
|
| 1386 |
"by_tier_open": {
|
| 1387 |
"Easy": {
|
| 1388 |
+
"score": 64.2,
|
| 1389 |
"n": 176
|
| 1390 |
},
|
| 1391 |
"Judge": {
|
| 1392 |
+
"score": 9.6,
|
| 1393 |
"n": 96
|
| 1394 |
},
|
| 1395 |
"Standard": {
|
| 1396 |
+
"score": 64.1,
|
| 1397 |
"n": 64
|
| 1398 |
}
|
| 1399 |
},
|
| 1400 |
"by_tier_sealed": {
|
| 1401 |
"Standard": {
|
| 1402 |
+
"score": 33.6,
|
| 1403 |
+
"n": 113
|
| 1404 |
},
|
| 1405 |
"Hard": {
|
| 1406 |
+
"score": 78.0,
|
| 1407 |
+
"n": 50
|
| 1408 |
+
},
|
| 1409 |
+
"Judge": {
|
| 1410 |
+
"score": 50.0,
|
| 1411 |
+
"n": 1
|
| 1412 |
}
|
| 1413 |
},
|
| 1414 |
"by_domain": {
|
| 1415 |
"incident_report": {
|
| 1416 |
+
"score": 50.0,
|
| 1417 |
"n": 160
|
| 1418 |
},
|
| 1419 |
"compute_ledger": {
|
| 1420 |
+
"score": 57.6,
|
| 1421 |
"n": 144
|
| 1422 |
},
|
| 1423 |
"cluster_throughput_comps": {
|
| 1424 |
+
"score": 1.3,
|
| 1425 |
"n": 16
|
| 1426 |
},
|
| 1427 |
"edge_traffic_forecast": {
|
| 1428 |
+
"score": 0.0,
|
| 1429 |
"n": 16
|
| 1430 |
},
|
| 1431 |
"squad_v2_open_qa": {
|
| 1432 |
"score": 80.0,
|
| 1433 |
"n": 60
|
| 1434 |
},
|
| 1435 |
+
"contract_extraction": {
|
| 1436 |
+
"score": 27.6,
|
| 1437 |
+
"n": 96
|
| 1438 |
+
},
|
| 1439 |
"adversarial_stress": {
|
| 1440 |
+
"score": 37.5,
|
| 1441 |
"n": 8
|
| 1442 |
}
|
| 1443 |
},
|
| 1444 |
+
"official_rank": 11
|
| 1445 |
},
|
| 1446 |
{
|
| 1447 |
"id": "claude-haiku-4.5-json",
|
|
|
|
| 1455 |
"runtime": "Hosted Cloud API ( ~38 output tokens/field )",
|
| 1456 |
"input_rate_per_1m_usd": 0.8,
|
| 1457 |
"output_rate_per_1m_usd": 4.0,
|
| 1458 |
+
"n_observations": 500,
|
| 1459 |
+
"input_tokens_total": 69943,
|
| 1460 |
+
"output_tokens_total": 19000,
|
| 1461 |
+
"usd_per_1k_decisions": 0.2639,
|
| 1462 |
"axes": {
|
| 1463 |
+
"intelligence": 95.4,
|
| 1464 |
"intelligence_ci95": [
|
| 1465 |
+
93.4,
|
| 1466 |
+
97.2
|
| 1467 |
],
|
| 1468 |
+
"calibration": 88.7,
|
| 1469 |
"speed": 41.8,
|
| 1470 |
+
"cost": 27.5
|
| 1471 |
},
|
| 1472 |
+
"capability_score": 92.1,
|
| 1473 |
+
"composite_score": 48.8,
|
| 1474 |
+
"open_set_score": 99.4,
|
| 1475 |
+
"sealed_set_score": 87.2,
|
| 1476 |
"calibration_details": {
|
| 1477 |
+
"null_auroc": 0.98592,
|
| 1478 |
+
"null_ece": 0.1566,
|
| 1479 |
+
"null_brier": 0.03222
|
| 1480 |
},
|
| 1481 |
"latency": {
|
| 1482 |
+
"mean_ms": 585.679,
|
| 1483 |
+
"p50_ms": 586.3918,
|
| 1484 |
+
"p95_ms": 717.8024,
|
| 1485 |
+
"p99_ms": 727.6919
|
| 1486 |
},
|
| 1487 |
"by_request_type": {
|
| 1488 |
"span_open": {
|
|
|
|
| 1491 |
},
|
| 1492 |
"math": {
|
| 1493 |
"score": 100.0,
|
| 1494 |
+
"n": 51
|
| 1495 |
},
|
| 1496 |
"choice": {
|
| 1497 |
"score": 100.0,
|
| 1498 |
+
"n": 41
|
| 1499 |
},
|
| 1500 |
"noul_open": {
|
| 1501 |
+
"score": 92.6,
|
| 1502 |
"n": 27
|
| 1503 |
},
|
| 1504 |
"score": {
|
|
|
|
| 1506 |
"n": 48
|
| 1507 |
},
|
| 1508 |
"span_sealed": {
|
| 1509 |
+
"score": 92.6,
|
| 1510 |
+
"n": 108
|
| 1511 |
},
|
| 1512 |
"noul_sealed": {
|
| 1513 |
+
"score": 70.5,
|
| 1514 |
+
"n": 44
|
| 1515 |
}
|
| 1516 |
},
|
| 1517 |
"by_tier_open": {
|
| 1518 |
"Easy": {
|
| 1519 |
+
"score": 98.9,
|
| 1520 |
"n": 176
|
| 1521 |
},
|
| 1522 |
"Judge": {
|
|
|
|
| 1530 |
},
|
| 1531 |
"by_tier_sealed": {
|
| 1532 |
"Standard": {
|
| 1533 |
+
"score": 92.9,
|
| 1534 |
+
"n": 113
|
| 1535 |
},
|
| 1536 |
"Hard": {
|
| 1537 |
+
"score": 74.0,
|
| 1538 |
+
"n": 50
|
| 1539 |
+
},
|
| 1540 |
+
"Judge": {
|
| 1541 |
+
"score": 100.0,
|
| 1542 |
+
"n": 1
|
| 1543 |
}
|
| 1544 |
},
|
| 1545 |
"by_domain": {
|
| 1546 |
"incident_report": {
|
| 1547 |
+
"score": 99.4,
|
| 1548 |
"n": 160
|
| 1549 |
},
|
| 1550 |
"compute_ledger": {
|
| 1551 |
+
"score": 99.3,
|
| 1552 |
"n": 144
|
| 1553 |
},
|
| 1554 |
"cluster_throughput_comps": {
|
|
|
|
| 1560 |
"n": 16
|
| 1561 |
},
|
| 1562 |
"squad_v2_open_qa": {
|
| 1563 |
+
"score": 78.3,
|
| 1564 |
"n": 60
|
| 1565 |
},
|
| 1566 |
+
"contract_extraction": {
|
| 1567 |
+
"score": 93.8,
|
| 1568 |
+
"n": 96
|
| 1569 |
+
},
|
| 1570 |
"adversarial_stress": {
|
| 1571 |
+
"score": 75.0,
|
| 1572 |
"n": 8
|
| 1573 |
}
|
| 1574 |
},
|
| 1575 |
+
"official_rank": 12
|
| 1576 |
},
|
| 1577 |
{
|
| 1578 |
"id": "bge-small-en-v1.5",
|
|
|
|
| 1586 |
"runtime": "PyTorch MPS/CPU",
|
| 1587 |
"input_rate_per_1m_usd": 0.01,
|
| 1588 |
"output_rate_per_1m_usd": 0.0,
|
| 1589 |
+
"n_observations": 500,
|
| 1590 |
+
"input_tokens_total": 35946,
|
| 1591 |
"output_tokens_total": 0,
|
| 1592 |
+
"usd_per_1k_decisions": 0.0007,
|
| 1593 |
"axes": {
|
| 1594 |
+
"intelligence": 14.7,
|
| 1595 |
"intelligence_ci95": [
|
| 1596 |
+
12.0,
|
| 1597 |
+
17.5
|
| 1598 |
],
|
| 1599 |
+
"calibration": 89.2,
|
| 1600 |
+
"speed": 64.5,
|
| 1601 |
+
"cost": 85.5
|
| 1602 |
},
|
| 1603 |
+
"capability_score": 52.0,
|
| 1604 |
+
"composite_score": 37.6,
|
| 1605 |
"open_set_score": 20.3,
|
| 1606 |
+
"sealed_set_score": 3.4,
|
| 1607 |
"calibration_details": {
|
| 1608 |
+
"null_auroc": 0.68694,
|
| 1609 |
+
"null_ece": 0.10548,
|
| 1610 |
+
"null_brier": 0.11334
|
| 1611 |
},
|
| 1612 |
"latency": {
|
| 1613 |
+
"mean_ms": 9.0856,
|
| 1614 |
+
"p50_ms": 8.0979,
|
| 1615 |
+
"p95_ms": 14.9339,
|
| 1616 |
+
"p99_ms": 40.3316
|
| 1617 |
},
|
| 1618 |
"by_request_type": {
|
| 1619 |
"span_open": {
|
|
|
|
| 1622 |
},
|
| 1623 |
"math": {
|
| 1624 |
"score": 0.0,
|
| 1625 |
+
"n": 51
|
| 1626 |
},
|
| 1627 |
"choice": {
|
| 1628 |
+
"score": 59.8,
|
| 1629 |
+
"n": 41
|
| 1630 |
},
|
| 1631 |
"noul_open": {
|
| 1632 |
"score": 44.4,
|
|
|
|
| 1637 |
"n": 48
|
| 1638 |
},
|
| 1639 |
"span_sealed": {
|
| 1640 |
+
"score": 2.3,
|
| 1641 |
+
"n": 108
|
| 1642 |
},
|
| 1643 |
"noul_sealed": {
|
| 1644 |
+
"score": 2.3,
|
| 1645 |
+
"n": 44
|
| 1646 |
}
|
| 1647 |
},
|
| 1648 |
"by_tier_open": {
|
|
|
|
| 1661 |
},
|
| 1662 |
"by_tier_sealed": {
|
| 1663 |
"Standard": {
|
| 1664 |
+
"score": 2.7,
|
| 1665 |
+
"n": 113
|
| 1666 |
},
|
| 1667 |
"Hard": {
|
| 1668 |
+
"score": 5.0,
|
| 1669 |
+
"n": 50
|
| 1670 |
+
},
|
| 1671 |
+
"Judge": {
|
| 1672 |
+
"score": 0.0,
|
| 1673 |
+
"n": 1
|
| 1674 |
}
|
| 1675 |
},
|
| 1676 |
"by_domain": {
|
|
|
|
| 1694 |
"score": 0.0,
|
| 1695 |
"n": 60
|
| 1696 |
},
|
| 1697 |
+
"contract_extraction": {
|
| 1698 |
+
"score": 3.1,
|
| 1699 |
+
"n": 96
|
| 1700 |
+
},
|
| 1701 |
"adversarial_stress": {
|
| 1702 |
"score": 31.2,
|
| 1703 |
"n": 8
|
| 1704 |
}
|
| 1705 |
},
|
| 1706 |
+
"official_rank": 13
|
| 1707 |
},
|
| 1708 |
{
|
| 1709 |
"id": "distilbert-base-control",
|
|
|
|
| 1717 |
"runtime": "PyTorch MPS/CPU",
|
| 1718 |
"input_rate_per_1m_usd": 0.015,
|
| 1719 |
"output_rate_per_1m_usd": 0.0,
|
| 1720 |
+
"n_observations": 500,
|
| 1721 |
+
"input_tokens_total": 35946,
|
| 1722 |
"output_tokens_total": 0,
|
| 1723 |
+
"usd_per_1k_decisions": 0.0011,
|
| 1724 |
"axes": {
|
| 1725 |
+
"intelligence": 14.2,
|
| 1726 |
"intelligence_ci95": [
|
| 1727 |
+
11.6,
|
| 1728 |
+
17.0
|
| 1729 |
],
|
| 1730 |
+
"calibration": 91.5,
|
| 1731 |
+
"speed": 66.0,
|
| 1732 |
+
"cost": 81.1
|
| 1733 |
},
|
| 1734 |
+
"capability_score": 52.9,
|
| 1735 |
+
"composite_score": 36.8,
|
| 1736 |
"open_set_score": 20.2,
|
| 1737 |
+
"sealed_set_score": 1.8,
|
| 1738 |
"calibration_details": {
|
| 1739 |
+
"null_auroc": 0.60728,
|
| 1740 |
+
"null_ece": 0.07034,
|
| 1741 |
+
"null_brier": 0.11186
|
| 1742 |
},
|
| 1743 |
"latency": {
|
| 1744 |
+
"mean_ms": 8.897,
|
| 1745 |
+
"p50_ms": 6.1334,
|
| 1746 |
+
"p95_ms": 19.077,
|
| 1747 |
+
"p99_ms": 26.1601
|
| 1748 |
},
|
| 1749 |
"by_request_type": {
|
| 1750 |
"span_open": {
|
|
|
|
| 1753 |
},
|
| 1754 |
"math": {
|
| 1755 |
"score": 0.0,
|
| 1756 |
+
"n": 51
|
| 1757 |
},
|
| 1758 |
"choice": {
|
| 1759 |
+
"score": 59.8,
|
| 1760 |
+
"n": 41
|
| 1761 |
},
|
| 1762 |
"noul_open": {
|
| 1763 |
"score": 0.0,
|
|
|
|
| 1768 |
"n": 48
|
| 1769 |
},
|
| 1770 |
"span_sealed": {
|
| 1771 |
+
"score": 0.9,
|
| 1772 |
+
"n": 108
|
| 1773 |
},
|
| 1774 |
"noul_sealed": {
|
| 1775 |
"score": 0.0,
|
| 1776 |
+
"n": 44
|
| 1777 |
}
|
| 1778 |
},
|
| 1779 |
"by_tier_open": {
|
|
|
|
| 1792 |
},
|
| 1793 |
"by_tier_sealed": {
|
| 1794 |
"Standard": {
|
| 1795 |
+
"score": 1.8,
|
| 1796 |
+
"n": 113
|
| 1797 |
},
|
| 1798 |
"Hard": {
|
| 1799 |
+
"score": 2.0,
|
| 1800 |
+
"n": 50
|
| 1801 |
+
},
|
| 1802 |
+
"Judge": {
|
| 1803 |
+
"score": 0.0,
|
| 1804 |
+
"n": 1
|
| 1805 |
}
|
| 1806 |
},
|
| 1807 |
"by_domain": {
|
|
|
|
| 1825 |
"score": 1.7,
|
| 1826 |
"n": 60
|
| 1827 |
},
|
| 1828 |
+
"contract_extraction": {
|
| 1829 |
+
"score": 1.0,
|
| 1830 |
+
"n": 96
|
| 1831 |
+
},
|
| 1832 |
"adversarial_stress": {
|
| 1833 |
"score": 12.5,
|
| 1834 |
"n": 8
|
| 1835 |
}
|
| 1836 |
},
|
| 1837 |
+
"official_rank": 14
|
| 1838 |
},
|
| 1839 |
{
|
| 1840 |
"id": "all-MiniLM-L12-v2",
|
|
|
|
| 1848 |
"runtime": "PyTorch MPS/CPU",
|
| 1849 |
"input_rate_per_1m_usd": 0.01,
|
| 1850 |
"output_rate_per_1m_usd": 0.0,
|
| 1851 |
+
"n_observations": 500,
|
| 1852 |
+
"input_tokens_total": 35946,
|
| 1853 |
"output_tokens_total": 0,
|
| 1854 |
+
"usd_per_1k_decisions": 0.0007,
|
| 1855 |
"axes": {
|
| 1856 |
+
"intelligence": 14.4,
|
| 1857 |
"intelligence_ci95": [
|
| 1858 |
+
11.7,
|
| 1859 |
+
17.5
|
| 1860 |
],
|
| 1861 |
+
"calibration": 73.5,
|
| 1862 |
+
"speed": 60.2,
|
| 1863 |
+
"cost": 85.5
|
| 1864 |
},
|
| 1865 |
+
"capability_score": 44.0,
|
| 1866 |
+
"composite_score": 35.9,
|
| 1867 |
"open_set_score": 17.2,
|
| 1868 |
+
"sealed_set_score": 8.8,
|
| 1869 |
"calibration_details": {
|
| 1870 |
+
"null_auroc": 0.58938,
|
| 1871 |
+
"null_ece": 0.28318,
|
| 1872 |
+
"null_brier": 0.23193
|
| 1873 |
},
|
| 1874 |
"latency": {
|
| 1875 |
+
"mean_ms": 18.2994,
|
| 1876 |
+
"p50_ms": 18.3563,
|
| 1877 |
+
"p95_ms": 25.4464,
|
| 1878 |
+
"p99_ms": 29.4546
|
| 1879 |
},
|
| 1880 |
"by_request_type": {
|
| 1881 |
"span_open": {
|
|
|
|
| 1884 |
},
|
| 1885 |
"math": {
|
| 1886 |
"score": 0.0,
|
| 1887 |
+
"n": 51
|
| 1888 |
},
|
| 1889 |
"choice": {
|
| 1890 |
+
"score": 28.0,
|
| 1891 |
+
"n": 41
|
| 1892 |
},
|
| 1893 |
"noul_open": {
|
| 1894 |
"score": 100.0,
|
|
|
|
| 1899 |
"n": 48
|
| 1900 |
},
|
| 1901 |
"span_sealed": {
|
| 1902 |
+
"score": 0.5,
|
| 1903 |
+
"n": 108
|
| 1904 |
},
|
| 1905 |
"noul_sealed": {
|
| 1906 |
+
"score": 29.5,
|
| 1907 |
+
"n": 44
|
| 1908 |
}
|
| 1909 |
},
|
| 1910 |
"by_tier_open": {
|
|
|
|
| 1923 |
},
|
| 1924 |
"by_tier_sealed": {
|
| 1925 |
"Standard": {
|
| 1926 |
+
"score": 0.9,
|
| 1927 |
+
"n": 113
|
| 1928 |
},
|
| 1929 |
"Hard": {
|
| 1930 |
+
"score": 27.0,
|
| 1931 |
+
"n": 50
|
| 1932 |
+
},
|
| 1933 |
+
"Judge": {
|
| 1934 |
+
"score": 0.0,
|
| 1935 |
+
"n": 1
|
| 1936 |
}
|
| 1937 |
},
|
| 1938 |
"by_domain": {
|
|
|
|
| 1956 |
"score": 1.7,
|
| 1957 |
"n": 60
|
| 1958 |
},
|
| 1959 |
+
"contract_extraction": {
|
| 1960 |
+
"score": 11.5,
|
| 1961 |
+
"n": 96
|
| 1962 |
+
},
|
| 1963 |
"adversarial_stress": {
|
| 1964 |
"score": 31.2,
|
| 1965 |
"n": 8
|
| 1966 |
}
|
| 1967 |
},
|
| 1968 |
+
"official_rank": 15
|
| 1969 |
},
|
| 1970 |
{
|
| 1971 |
"id": "all-mpnet-base-v2",
|
|
|
|
| 1979 |
"runtime": "PyTorch MPS/CPU",
|
| 1980 |
"input_rate_per_1m_usd": 0.022,
|
| 1981 |
"output_rate_per_1m_usd": 0.0,
|
| 1982 |
+
"n_observations": 500,
|
| 1983 |
+
"input_tokens_total": 35946,
|
| 1984 |
"output_tokens_total": 0,
|
| 1985 |
+
"usd_per_1k_decisions": 0.0016,
|
| 1986 |
"axes": {
|
| 1987 |
+
"intelligence": 13.6,
|
| 1988 |
"intelligence_ci95": [
|
| 1989 |
10.9,
|
| 1990 |
+
16.7
|
| 1991 |
],
|
| 1992 |
+
"calibration": 71.0,
|
| 1993 |
"speed": 62.8,
|
| 1994 |
+
"cost": 77.4
|
| 1995 |
},
|
| 1996 |
+
"capability_score": 42.3,
|
| 1997 |
+
"composite_score": 34.3,
|
| 1998 |
"open_set_score": 16.2,
|
| 1999 |
+
"sealed_set_score": 8.2,
|
| 2000 |
"calibration_details": {
|
| 2001 |
+
"null_auroc": 0.58731,
|
| 2002 |
+
"null_ece": 0.30424,
|
| 2003 |
+
"null_brier": 0.26407
|
| 2004 |
},
|
| 2005 |
"latency": {
|
| 2006 |
+
"mean_ms": 14.1298,
|
| 2007 |
+
"p50_ms": 11.2628,
|
| 2008 |
+
"p95_ms": 32.6527,
|
| 2009 |
+
"p99_ms": 52.232
|
| 2010 |
},
|
| 2011 |
"by_request_type": {
|
| 2012 |
"span_open": {
|
|
|
|
| 2015 |
},
|
| 2016 |
"math": {
|
| 2017 |
"score": 0.0,
|
| 2018 |
+
"n": 51
|
| 2019 |
},
|
| 2020 |
"choice": {
|
| 2021 |
+
"score": 28.0,
|
| 2022 |
+
"n": 41
|
| 2023 |
},
|
| 2024 |
"noul_open": {
|
| 2025 |
"score": 100.0,
|
|
|
|
| 2030 |
"n": 48
|
| 2031 |
},
|
| 2032 |
"span_sealed": {
|
| 2033 |
+
"score": 0.5,
|
| 2034 |
+
"n": 108
|
| 2035 |
},
|
| 2036 |
"noul_sealed": {
|
| 2037 |
+
"score": 27.3,
|
| 2038 |
+
"n": 44
|
| 2039 |
}
|
| 2040 |
},
|
| 2041 |
"by_tier_open": {
|
|
|
|
| 2054 |
},
|
| 2055 |
"by_tier_sealed": {
|
| 2056 |
"Standard": {
|
| 2057 |
+
"score": 0.9,
|
| 2058 |
+
"n": 113
|
| 2059 |
},
|
| 2060 |
"Hard": {
|
| 2061 |
+
"score": 25.0,
|
| 2062 |
+
"n": 50
|
| 2063 |
+
},
|
| 2064 |
+
"Judge": {
|
| 2065 |
+
"score": 0.0,
|
| 2066 |
+
"n": 1
|
| 2067 |
}
|
| 2068 |
},
|
| 2069 |
"by_domain": {
|
|
|
|
| 2087 |
"score": 0.0,
|
| 2088 |
"n": 60
|
| 2089 |
},
|
| 2090 |
+
"contract_extraction": {
|
| 2091 |
+
"score": 11.5,
|
| 2092 |
+
"n": 96
|
| 2093 |
+
},
|
| 2094 |
"adversarial_stress": {
|
| 2095 |
"score": 31.2,
|
| 2096 |
"n": 8
|
| 2097 |
}
|
| 2098 |
},
|
| 2099 |
+
"official_rank": 16
|
| 2100 |
},
|
| 2101 |
{
|
| 2102 |
"id": "paraphrase-albert-small-v2",
|
|
|
|
| 2110 |
"runtime": "PyTorch MPS/CPU",
|
| 2111 |
"input_rate_per_1m_usd": 0.005,
|
| 2112 |
"output_rate_per_1m_usd": 0.0,
|
| 2113 |
+
"n_observations": 500,
|
| 2114 |
+
"input_tokens_total": 35946,
|
| 2115 |
"output_tokens_total": 0,
|
| 2116 |
+
"usd_per_1k_decisions": 0.0004,
|
| 2117 |
"axes": {
|
| 2118 |
+
"intelligence": 12.5,
|
| 2119 |
"intelligence_ci95": [
|
| 2120 |
+
10.0,
|
| 2121 |
+
15.2
|
| 2122 |
],
|
| 2123 |
+
"calibration": 76.6,
|
| 2124 |
+
"speed": 65.9,
|
| 2125 |
+
"cost": 91.0
|
| 2126 |
},
|
| 2127 |
+
"capability_score": 44.5,
|
| 2128 |
+
"composite_score": 33.6,
|
| 2129 |
"open_set_score": 15.3,
|
| 2130 |
+
"sealed_set_score": 6.7,
|
| 2131 |
"calibration_details": {
|
| 2132 |
+
"null_auroc": 0.58482,
|
| 2133 |
+
"null_ece": 0.24547,
|
| 2134 |
+
"null_brier": 0.21391
|
| 2135 |
},
|
| 2136 |
"latency": {
|
| 2137 |
+
"mean_ms": 6.9075,
|
| 2138 |
+
"p50_ms": 6.2627,
|
| 2139 |
+
"p95_ms": 7.7752,
|
| 2140 |
+
"p99_ms": 30.2162
|
| 2141 |
},
|
| 2142 |
"by_request_type": {
|
| 2143 |
"span_open": {
|
|
|
|
| 2145 |
"n": 181
|
| 2146 |
},
|
| 2147 |
"math": {
|
| 2148 |
+
"score": 1.0,
|
| 2149 |
+
"n": 51
|
| 2150 |
},
|
| 2151 |
"choice": {
|
| 2152 |
+
"score": 35.4,
|
| 2153 |
+
"n": 41
|
| 2154 |
},
|
| 2155 |
"noul_open": {
|
| 2156 |
"score": 51.9,
|
|
|
|
| 2161 |
"n": 48
|
| 2162 |
},
|
| 2163 |
"span_sealed": {
|
| 2164 |
+
"score": 2.3,
|
| 2165 |
+
"n": 108
|
| 2166 |
},
|
| 2167 |
"noul_sealed": {
|
| 2168 |
+
"score": 15.9,
|
| 2169 |
+
"n": 44
|
| 2170 |
}
|
| 2171 |
},
|
| 2172 |
"by_tier_open": {
|
|
|
|
| 2185 |
},
|
| 2186 |
"by_tier_sealed": {
|
| 2187 |
"Standard": {
|
| 2188 |
+
"score": 2.7,
|
| 2189 |
+
"n": 113
|
| 2190 |
},
|
| 2191 |
"Hard": {
|
| 2192 |
+
"score": 15.0,
|
| 2193 |
+
"n": 50
|
| 2194 |
+
},
|
| 2195 |
+
"Judge": {
|
| 2196 |
+
"score": 50.0,
|
| 2197 |
+
"n": 1
|
| 2198 |
}
|
| 2199 |
},
|
| 2200 |
"by_domain": {
|
|
|
|
| 2218 |
"score": 0.0,
|
| 2219 |
"n": 60
|
| 2220 |
},
|
| 2221 |
+
"contract_extraction": {
|
| 2222 |
+
"score": 9.9,
|
| 2223 |
+
"n": 96
|
| 2224 |
+
},
|
| 2225 |
"adversarial_stress": {
|
| 2226 |
"score": 18.8,
|
| 2227 |
"n": 8
|
| 2228 |
}
|
| 2229 |
},
|
| 2230 |
+
"official_rank": 17
|
| 2231 |
},
|
| 2232 |
{
|
| 2233 |
"id": "paraphrase-MiniLM-L3-v2",
|
|
|
|
| 2241 |
"runtime": "PyTorch MPS/CPU",
|
| 2242 |
"input_rate_per_1m_usd": 0.005,
|
| 2243 |
"output_rate_per_1m_usd": 0.0,
|
| 2244 |
+
"n_observations": 500,
|
| 2245 |
+
"input_tokens_total": 35946,
|
| 2246 |
"output_tokens_total": 0,
|
| 2247 |
+
"usd_per_1k_decisions": 0.0004,
|
| 2248 |
"axes": {
|
| 2249 |
+
"intelligence": 11.1,
|
| 2250 |
"intelligence_ci95": [
|
| 2251 |
+
8.7,
|
| 2252 |
+
13.8
|
| 2253 |
],
|
| 2254 |
+
"calibration": 72.3,
|
| 2255 |
+
"speed": 68.9,
|
| 2256 |
+
"cost": 91.0
|
| 2257 |
},
|
| 2258 |
+
"capability_score": 41.7,
|
| 2259 |
+
"composite_score": 30.9,
|
| 2260 |
"open_set_score": 12.9,
|
| 2261 |
+
"sealed_set_score": 7.6,
|
| 2262 |
"calibration_details": {
|
| 2263 |
+
"null_auroc": 0.58932,
|
| 2264 |
+
"null_ece": 0.29025,
|
| 2265 |
+
"null_brier": 0.25176
|
| 2266 |
},
|
| 2267 |
"latency": {
|
| 2268 |
+
"mean_ms": 5.6642,
|
| 2269 |
+
"p50_ms": 3.5192,
|
| 2270 |
+
"p95_ms": 15.1691,
|
| 2271 |
+
"p99_ms": 19.9379
|
| 2272 |
},
|
| 2273 |
"by_request_type": {
|
| 2274 |
"span_open": {
|
|
|
|
| 2277 |
},
|
| 2278 |
"math": {
|
| 2279 |
"score": 0.0,
|
| 2280 |
+
"n": 51
|
| 2281 |
},
|
| 2282 |
"choice": {
|
| 2283 |
+
"score": 28.0,
|
| 2284 |
+
"n": 41
|
| 2285 |
},
|
| 2286 |
"noul_open": {
|
| 2287 |
"score": 44.4,
|
|
|
|
| 2292 |
"n": 48
|
| 2293 |
},
|
| 2294 |
"span_sealed": {
|
| 2295 |
+
"score": 0.5,
|
| 2296 |
+
"n": 108
|
| 2297 |
},
|
| 2298 |
"noul_sealed": {
|
| 2299 |
+
"score": 25.0,
|
| 2300 |
+
"n": 44
|
| 2301 |
}
|
| 2302 |
},
|
| 2303 |
"by_tier_open": {
|
|
|
|
| 2316 |
},
|
| 2317 |
"by_tier_sealed": {
|
| 2318 |
"Standard": {
|
| 2319 |
+
"score": 0.9,
|
| 2320 |
+
"n": 113
|
| 2321 |
},
|
| 2322 |
"Hard": {
|
| 2323 |
+
"score": 23.0,
|
| 2324 |
+
"n": 50
|
| 2325 |
+
},
|
| 2326 |
+
"Judge": {
|
| 2327 |
+
"score": 0.0,
|
| 2328 |
+
"n": 1
|
| 2329 |
}
|
| 2330 |
},
|
| 2331 |
"by_domain": {
|
|
|
|
| 2349 |
"score": 1.7,
|
| 2350 |
"n": 60
|
| 2351 |
},
|
| 2352 |
+
"contract_extraction": {
|
| 2353 |
+
"score": 10.4,
|
| 2354 |
+
"n": 96
|
| 2355 |
+
},
|
| 2356 |
"adversarial_stress": {
|
| 2357 |
"score": 18.8,
|
| 2358 |
"n": 8
|
| 2359 |
}
|
| 2360 |
},
|
| 2361 |
+
"official_rank": 18
|
| 2362 |
},
|
| 2363 |
{
|
| 2364 |
"id": "all-MiniLM-L6-v2",
|
|
|
|
| 2372 |
"runtime": "PyTorch MPS/CPU",
|
| 2373 |
"input_rate_per_1m_usd": 0.007,
|
| 2374 |
"output_rate_per_1m_usd": 0.0,
|
| 2375 |
+
"n_observations": 500,
|
| 2376 |
+
"input_tokens_total": 35946,
|
| 2377 |
"output_tokens_total": 0,
|
| 2378 |
+
"usd_per_1k_decisions": 0.0005,
|
| 2379 |
"axes": {
|
| 2380 |
+
"intelligence": 10.0,
|
| 2381 |
"intelligence_ci95": [
|
| 2382 |
+
7.7,
|
| 2383 |
+
12.3
|
| 2384 |
],
|
| 2385 |
+
"calibration": 70.9,
|
| 2386 |
+
"speed": 62.3,
|
| 2387 |
+
"cost": 88.8
|
| 2388 |
},
|
| 2389 |
+
"capability_score": 40.5,
|
| 2390 |
+
"composite_score": 28.3,
|
| 2391 |
"open_set_score": 11.2,
|
| 2392 |
+
"sealed_set_score": 7.6,
|
| 2393 |
"calibration_details": {
|
| 2394 |
+
"null_auroc": 0.56307,
|
| 2395 |
+
"null_ece": 0.30616,
|
| 2396 |
+
"null_brier": 0.26323
|
| 2397 |
},
|
| 2398 |
"latency": {
|
| 2399 |
+
"mean_ms": 13.0226,
|
| 2400 |
+
"p50_ms": 12.3218,
|
| 2401 |
+
"p95_ms": 20.8164,
|
| 2402 |
+
"p99_ms": 24.6272
|
| 2403 |
},
|
| 2404 |
"by_request_type": {
|
| 2405 |
"span_open": {
|
|
|
|
| 2408 |
},
|
| 2409 |
"math": {
|
| 2410 |
"score": 0.0,
|
| 2411 |
+
"n": 51
|
| 2412 |
},
|
| 2413 |
"choice": {
|
| 2414 |
+
"score": 28.0,
|
| 2415 |
+
"n": 41
|
| 2416 |
},
|
| 2417 |
"noul_open": {
|
| 2418 |
"score": 44.4,
|
|
|
|
| 2423 |
"n": 48
|
| 2424 |
},
|
| 2425 |
"span_sealed": {
|
| 2426 |
+
"score": 0.5,
|
| 2427 |
+
"n": 108
|
| 2428 |
},
|
| 2429 |
"noul_sealed": {
|
| 2430 |
+
"score": 25.0,
|
| 2431 |
+
"n": 44
|
| 2432 |
}
|
| 2433 |
},
|
| 2434 |
"by_tier_open": {
|
|
|
|
| 2447 |
},
|
| 2448 |
"by_tier_sealed": {
|
| 2449 |
"Standard": {
|
| 2450 |
+
"score": 0.9,
|
| 2451 |
+
"n": 113
|
| 2452 |
},
|
| 2453 |
"Hard": {
|
| 2454 |
+
"score": 23.0,
|
| 2455 |
+
"n": 50
|
| 2456 |
+
},
|
| 2457 |
+
"Judge": {
|
| 2458 |
+
"score": 0.0,
|
| 2459 |
+
"n": 1
|
| 2460 |
}
|
| 2461 |
},
|
| 2462 |
"by_domain": {
|
|
|
|
| 2480 |
"score": 1.7,
|
| 2481 |
"n": 60
|
| 2482 |
},
|
| 2483 |
+
"contract_extraction": {
|
| 2484 |
+
"score": 10.4,
|
| 2485 |
+
"n": 96
|
| 2486 |
+
},
|
| 2487 |
"adversarial_stress": {
|
| 2488 |
"score": 18.8,
|
| 2489 |
"n": 8
|
| 2490 |
}
|
| 2491 |
},
|
| 2492 |
+
"official_rank": 19
|
| 2493 |
}
|
| 2494 |
],
|
| 2495 |
"tiers": {
|
| 2496 |
"tier_1a_pytorch_fp32": {
|
| 2497 |
+
"id": "nullspan-fp32",
|
| 2498 |
+
"name": "nullspan-344M (PyTorch FP32)",
|
| 2499 |
"author": "gnumanth",
|
| 2500 |
+
"system_type": "System One Encoder (5-Head, 512-ctx)",
|
| 2501 |
"open_weights": true,
|
| 2502 |
+
"params_m": 344.2,
|
| 2503 |
+
"footprint_mb": 1313.1,
|
| 2504 |
+
"base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter",
|
| 2505 |
+
"runtime": "PyTorch MPS/CUDA/CPU FP32",
|
| 2506 |
+
"input_rate_per_1m_usd": 0.012,
|
| 2507 |
"output_rate_per_1m_usd": 0.0,
|
| 2508 |
+
"n_observations": 500,
|
| 2509 |
+
"input_tokens_total": 69943,
|
| 2510 |
"output_tokens_total": 0,
|
| 2511 |
+
"usd_per_1k_decisions": 0.0017,
|
| 2512 |
"axes": {
|
| 2513 |
+
"intelligence": 98.0,
|
| 2514 |
"intelligence_ci95": [
|
| 2515 |
+
96.8,
|
| 2516 |
+
99.2
|
| 2517 |
],
|
| 2518 |
+
"calibration": 98.8,
|
| 2519 |
+
"speed": 57.2,
|
| 2520 |
+
"cost": 76.8
|
| 2521 |
},
|
| 2522 |
+
"capability_score": 98.4,
|
| 2523 |
+
"composite_score": 78.7,
|
| 2524 |
"open_set_score": 100.0,
|
| 2525 |
+
"sealed_set_score": 93.9,
|
| 2526 |
"calibration_details": {
|
| 2527 |
+
"null_auroc": 0.99829,
|
| 2528 |
+
"null_ece": 0.01215,
|
| 2529 |
+
"null_brier": 0.01245
|
| 2530 |
},
|
| 2531 |
"latency": {
|
| 2532 |
+
"mean_ms": 47.0471,
|
| 2533 |
+
"p50_ms": 32.3672,
|
| 2534 |
+
"p95_ms": 91.8385,
|
| 2535 |
+
"p99_ms": 108.335
|
| 2536 |
},
|
| 2537 |
"by_request_type": {
|
| 2538 |
"span_open": {
|
|
|
|
| 2541 |
},
|
| 2542 |
"math": {
|
| 2543 |
"score": 100.0,
|
| 2544 |
+
"n": 51
|
| 2545 |
},
|
| 2546 |
"choice": {
|
| 2547 |
"score": 100.0,
|
| 2548 |
+
"n": 41
|
| 2549 |
},
|
| 2550 |
"noul_open": {
|
| 2551 |
"score": 100.0,
|
|
|
|
| 2556 |
"n": 48
|
| 2557 |
},
|
| 2558 |
"span_sealed": {
|
| 2559 |
+
"score": 96.3,
|
| 2560 |
+
"n": 108
|
| 2561 |
},
|
| 2562 |
"noul_sealed": {
|
| 2563 |
+
"score": 86.4,
|
| 2564 |
+
"n": 44
|
| 2565 |
}
|
| 2566 |
},
|
| 2567 |
"by_tier_open": {
|
|
|
|
| 2580 |
},
|
| 2581 |
"by_tier_sealed": {
|
| 2582 |
"Standard": {
|
| 2583 |
+
"score": 96.5,
|
| 2584 |
+
"n": 113
|
| 2585 |
},
|
| 2586 |
"Hard": {
|
| 2587 |
+
"score": 88.0,
|
| 2588 |
+
"n": 50
|
| 2589 |
+
},
|
| 2590 |
+
"Judge": {
|
| 2591 |
"score": 100.0,
|
| 2592 |
+
"n": 1
|
| 2593 |
}
|
| 2594 |
},
|
| 2595 |
"by_domain": {
|
|
|
|
| 2610 |
"n": 16
|
| 2611 |
},
|
| 2612 |
"squad_v2_open_qa": {
|
| 2613 |
+
"score": 83.3,
|
| 2614 |
"n": 60
|
| 2615 |
},
|
| 2616 |
+
"contract_extraction": {
|
| 2617 |
+
"score": 100.0,
|
| 2618 |
+
"n": 96
|
| 2619 |
+
},
|
| 2620 |
"adversarial_stress": {
|
| 2621 |
"score": 100.0,
|
| 2622 |
"n": 8
|
|
|
|
| 2625 |
"official_rank": 3
|
| 2626 |
},
|
| 2627 |
"tier_1b_onnx_int8": {
|
| 2628 |
+
"id": "nullspan-onnx-int8",
|
| 2629 |
+
"name": "nullspan-344M-Int8 (ONNX WASM/CPU)",
|
| 2630 |
"author": "gnumanth",
|
| 2631 |
+
"system_type": "Quantized ONNX WASM/CPU (512-ctx)",
|
| 2632 |
"open_weights": true,
|
| 2633 |
+
"params_m": 344.2,
|
| 2634 |
+
"footprint_mb": 330.1,
|
| 2635 |
"base_model": "gnumanth/nullspan (Int8 Dynamic)",
|
| 2636 |
"runtime": "ONNX Runtime Int8 (Browser WASM / Edge)",
|
| 2637 |
+
"input_rate_per_1m_usd": 0.006,
|
| 2638 |
"output_rate_per_1m_usd": 0.0,
|
| 2639 |
+
"n_observations": 500,
|
| 2640 |
+
"input_tokens_total": 69943,
|
| 2641 |
"output_tokens_total": 0,
|
| 2642 |
"usd_per_1k_decisions": 0.0008,
|
| 2643 |
"axes": {
|
| 2644 |
+
"intelligence": 97.8,
|
| 2645 |
"intelligence_ci95": [
|
| 2646 |
+
96.6,
|
| 2647 |
+
99.0
|
| 2648 |
],
|
| 2649 |
+
"calibration": 98.9,
|
| 2650 |
+
"speed": 53.2,
|
| 2651 |
"cost": 84.2
|
| 2652 |
},
|
| 2653 |
+
"capability_score": 98.3,
|
| 2654 |
+
"composite_score": 78.4,
|
| 2655 |
"open_set_score": 100.0,
|
| 2656 |
+
"sealed_set_score": 93.3,
|
| 2657 |
"calibration_details": {
|
| 2658 |
+
"null_auroc": 0.99737,
|
| 2659 |
+
"null_ece": 0.0095,
|
| 2660 |
+
"null_brier": 0.01314
|
| 2661 |
},
|
| 2662 |
"latency": {
|
| 2663 |
+
"mean_ms": 107.1327,
|
| 2664 |
+
"p50_ms": 68.4825,
|
| 2665 |
+
"p95_ms": 262.6889,
|
| 2666 |
+
"p99_ms": 311.1327
|
| 2667 |
},
|
| 2668 |
"by_request_type": {
|
| 2669 |
"span_open": {
|
|
|
|
| 2672 |
},
|
| 2673 |
"math": {
|
| 2674 |
"score": 100.0,
|
| 2675 |
+
"n": 51
|
| 2676 |
},
|
| 2677 |
"choice": {
|
| 2678 |
"score": 100.0,
|
| 2679 |
+
"n": 41
|
| 2680 |
},
|
| 2681 |
"noul_open": {
|
| 2682 |
"score": 100.0,
|
|
|
|
| 2687 |
"n": 48
|
| 2688 |
},
|
| 2689 |
"span_sealed": {
|
| 2690 |
+
"score": 95.4,
|
| 2691 |
+
"n": 108
|
| 2692 |
},
|
| 2693 |
"noul_sealed": {
|
| 2694 |
+
"score": 86.4,
|
| 2695 |
+
"n": 44
|
| 2696 |
}
|
| 2697 |
},
|
| 2698 |
"by_tier_open": {
|
|
|
|
| 2711 |
},
|
| 2712 |
"by_tier_sealed": {
|
| 2713 |
"Standard": {
|
| 2714 |
+
"score": 95.6,
|
| 2715 |
+
"n": 113
|
| 2716 |
},
|
| 2717 |
"Hard": {
|
| 2718 |
+
"score": 88.0,
|
| 2719 |
+
"n": 50
|
| 2720 |
+
},
|
| 2721 |
+
"Judge": {
|
| 2722 |
+
"score": 100.0,
|
| 2723 |
+
"n": 1
|
| 2724 |
}
|
| 2725 |
},
|
| 2726 |
"by_domain": {
|
|
|
|
| 2741 |
"n": 16
|
| 2742 |
},
|
| 2743 |
"squad_v2_open_qa": {
|
| 2744 |
+
"score": 81.7,
|
| 2745 |
"n": 60
|
| 2746 |
},
|
| 2747 |
+
"contract_extraction": {
|
| 2748 |
+
"score": 100.0,
|
| 2749 |
+
"n": 96
|
| 2750 |
+
},
|
| 2751 |
"adversarial_stress": {
|
| 2752 |
"score": 100.0,
|
| 2753 |
"n": 8
|
|
|
|
| 2763 |
"open_weights": true,
|
| 2764 |
"params_m": 0.0,
|
| 2765 |
"footprint_mb": 0.0,
|
| 2766 |
+
"base_model": "Deterministic Clause + Lexical Pointer",
|
| 2767 |
"runtime": "Pure Python / Node / Browser JS",
|
| 2768 |
"input_rate_per_1m_usd": 0.0008,
|
| 2769 |
"output_rate_per_1m_usd": 0.0,
|
| 2770 |
+
"n_observations": 500,
|
| 2771 |
+
"input_tokens_total": 69943,
|
| 2772 |
"output_tokens_total": 0,
|
| 2773 |
"usd_per_1k_decisions": 0.0001,
|
| 2774 |
"axes": {
|
| 2775 |
+
"intelligence": 90.8,
|
| 2776 |
"intelligence_ci95": [
|
| 2777 |
+
88.2,
|
| 2778 |
+
93.4
|
| 2779 |
],
|
| 2780 |
+
"calibration": 95.0,
|
| 2781 |
+
"speed": 98.7,
|
| 2782 |
"cost": 99.8
|
| 2783 |
},
|
| 2784 |
+
"capability_score": 92.9,
|
| 2785 |
+
"composite_score": 95.9,
|
| 2786 |
"open_set_score": 100.0,
|
| 2787 |
+
"sealed_set_score": 72.0,
|
| 2788 |
"calibration_details": {
|
| 2789 |
+
"null_auroc": 0.98239,
|
| 2790 |
+
"null_ece": 0.04866,
|
| 2791 |
+
"null_brier": 0.05142
|
| 2792 |
},
|
| 2793 |
"latency": {
|
| 2794 |
+
"mean_ms": 0.0632,
|
| 2795 |
+
"p50_ms": 0.0127,
|
| 2796 |
+
"p95_ms": 0.2868,
|
| 2797 |
+
"p99_ms": 0.3604
|
| 2798 |
},
|
| 2799 |
"by_request_type": {
|
| 2800 |
"span_open": {
|
|
|
|
| 2803 |
},
|
| 2804 |
"math": {
|
| 2805 |
"score": 100.0,
|
| 2806 |
+
"n": 51
|
| 2807 |
},
|
| 2808 |
"choice": {
|
| 2809 |
"score": 100.0,
|
| 2810 |
+
"n": 41
|
| 2811 |
},
|
| 2812 |
"noul_open": {
|
| 2813 |
"score": 100.0,
|
|
|
|
| 2818 |
"n": 48
|
| 2819 |
},
|
| 2820 |
"span_sealed": {
|
| 2821 |
+
"score": 78.7,
|
| 2822 |
+
"n": 108
|
| 2823 |
},
|
| 2824 |
"noul_sealed": {
|
| 2825 |
+
"score": 47.7,
|
| 2826 |
+
"n": 44
|
| 2827 |
}
|
| 2828 |
},
|
| 2829 |
"by_tier_open": {
|
|
|
|
| 2842 |
},
|
| 2843 |
"by_tier_sealed": {
|
| 2844 |
"Standard": {
|
| 2845 |
+
"score": 79.6,
|
| 2846 |
+
"n": 113
|
| 2847 |
},
|
| 2848 |
"Hard": {
|
| 2849 |
+
"score": 54.0,
|
| 2850 |
+
"n": 50
|
| 2851 |
+
},
|
| 2852 |
+
"Judge": {
|
| 2853 |
+
"score": 100.0,
|
| 2854 |
+
"n": 1
|
| 2855 |
}
|
| 2856 |
},
|
| 2857 |
"by_domain": {
|
|
|
|
| 2875 |
"score": 23.3,
|
| 2876 |
"n": 60
|
| 2877 |
},
|
| 2878 |
+
"contract_extraction": {
|
| 2879 |
+
"score": 100.0,
|
| 2880 |
+
"n": 96
|
| 2881 |
+
},
|
| 2882 |
"adversarial_stress": {
|
| 2883 |
"score": 100.0,
|
| 2884 |
"n": 8
|
bench.html
CHANGED
|
@@ -185,6 +185,7 @@
|
|
| 185 |
border: 1.5px solid var(--border);
|
| 186 |
border-radius: 6px;
|
| 187 |
overflow: hidden;
|
|
|
|
| 188 |
}
|
| 189 |
|
| 190 |
.section-header {
|
|
@@ -279,6 +280,7 @@
|
|
| 279 |
border-radius: 6px;
|
| 280 |
padding: 14px;
|
| 281 |
background: var(--bg);
|
|
|
|
| 282 |
}
|
| 283 |
|
| 284 |
.chart-box h3 {
|
|
@@ -386,14 +388,78 @@
|
|
| 386 |
text-align: center !important;
|
| 387 |
font-weight: 700;
|
| 388 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 389 |
</style>
|
| 390 |
</head>
|
| 391 |
<body>
|
| 392 |
<header>
|
| 393 |
<div class="brand">
|
| 394 |
<a href="./index.html" class="brand-mark">nullspan</a>
|
| 395 |
-
<span class="badge" style="background: var(--accent-span);">NullspanBench v3.
|
| 396 |
-
<span class="badge" id="draw-badge">S+P (
|
| 397 |
</div>
|
| 398 |
<nav class="links">
|
| 399 |
<a href="./index.html">← Interactive Playground</a>
|
|
@@ -409,13 +475,13 @@
|
|
| 409 |
<div>
|
| 410 |
<h1>NullspanBench — Decision & Extraction Model Leaderboard</h1>
|
| 411 |
<p>
|
| 412 |
-
Following the <strong>Benchmark Heaven (JevBench)</strong> evaluation protocol: comparing
|
| 413 |
</p>
|
| 414 |
</div>
|
| 415 |
<div class="btn-group">
|
| 416 |
<span class="btn-label">Pool Filter:</span>
|
| 417 |
-
<button class="ctrl-btn" id="btn-pool-open" type="button">Open Weights Only
|
| 418 |
-
<button class="ctrl-btn active" id="btn-pool-all" type="button">Show All Mixed (+
|
| 419 |
</div>
|
| 420 |
</section>
|
| 421 |
|
|
@@ -456,7 +522,7 @@
|
|
| 456 |
<section class="section-card">
|
| 457 |
<div class="section-header">
|
| 458 |
<span>3. Composite Score Leaderboard · Four Axes (0–100) Weighted Harmonic Mean</span>
|
| 459 |
-
<span id="ranked-count-label">
|
| 460 |
</div>
|
| 461 |
<div class="section-body">
|
| 462 |
<div class="filter-bar" style="margin-bottom: 12px;">
|
|
@@ -568,7 +634,7 @@
|
|
| 568 |
<section class="section-card">
|
| 569 |
<div class="section-header">
|
| 570 |
<span>5. Competence Heatmap by Request Type, Difficulty Tier & Subject Domain</span>
|
| 571 |
-
<span>Greener = Stronger Within Column · Exact Observation Counts (n=
|
| 572 |
</div>
|
| 573 |
<div class="section-body">
|
| 574 |
<div class="table-wrap">
|
|
@@ -590,6 +656,7 @@
|
|
| 590 |
<th>SRE Incident</th>
|
| 591 |
<th>Cloud Invoice</th>
|
| 592 |
<th>SQuAD v2.0 QA</th>
|
|
|
|
| 593 |
<th>Adversarial</th>
|
| 594 |
</tr>
|
| 595 |
</thead>
|
|
@@ -890,12 +957,13 @@
|
|
| 890 |
['cluster_throughput_comps', 'Cluster Comps'],
|
| 891 |
['edge_traffic_forecast', 'Edge Forecast'],
|
| 892 |
['squad_v2_open_qa', 'SQuAD v2.0'],
|
|
|
|
| 893 |
['adversarial_stress', 'Adversarial']
|
| 894 |
];
|
| 895 |
const domA = domKeys.map(([k]) => sysA.by_domain[k]?.score ?? 0);
|
| 896 |
const domB = domKeys.map(([k]) => sysB.by_domain[k]?.score ?? 0);
|
| 897 |
document.getElementById('radar-desc-domain').textContent =
|
| 898 |
-
`SQuAD v2.0: ${domA[4]}% vs ${domB[4]}% ·
|
| 899 |
drawRadarSvg('radar-domain-svg', domKeys.map(x => x[1]), domA, domB);
|
| 900 |
}
|
| 901 |
|
|
@@ -922,6 +990,7 @@
|
|
| 922 |
${cell(s.by_domain.incident_report)}
|
| 923 |
${cell(s.by_domain.compute_ledger)}
|
| 924 |
${cell(s.by_domain.squad_v2_open_qa)}
|
|
|
|
| 925 |
${cell(s.by_domain.adversarial_stress)}
|
| 926 |
</tr>
|
| 927 |
`).join('');
|
|
@@ -947,7 +1016,7 @@
|
|
| 947 |
const selB = document.getElementById('select-sys-b');
|
| 948 |
selA.innerHTML = BENCH_DATA.systems.map(s => `<option value="${s.id}">${s.name}</option>`).join('');
|
| 949 |
selB.innerHTML = BENCH_DATA.systems.map(s => `<option value="${s.id}">${s.name}</option>`).join('');
|
| 950 |
-
selA.value = 'nullspan-
|
| 951 |
selB.value = 'cloudflare-clairic-clef-live';
|
| 952 |
|
| 953 |
selA.addEventListener('change', renderComparisonRadars);
|
|
|
|
| 185 |
border: 1.5px solid var(--border);
|
| 186 |
border-radius: 6px;
|
| 187 |
overflow: hidden;
|
| 188 |
+
min-width: 0;
|
| 189 |
}
|
| 190 |
|
| 191 |
.section-header {
|
|
|
|
| 280 |
border-radius: 6px;
|
| 281 |
padding: 14px;
|
| 282 |
background: var(--bg);
|
| 283 |
+
min-width: 0;
|
| 284 |
}
|
| 285 |
|
| 286 |
.chart-box h3 {
|
|
|
|
| 388 |
text-align: center !important;
|
| 389 |
font-weight: 700;
|
| 390 |
}
|
| 391 |
+
|
| 392 |
+
@media (max-width: 760px) {
|
| 393 |
+
header {
|
| 394 |
+
position: static;
|
| 395 |
+
padding: 12px 14px;
|
| 396 |
+
gap: 10px;
|
| 397 |
+
}
|
| 398 |
+
.brand .badge {
|
| 399 |
+
font-size: 0.65rem;
|
| 400 |
+
padding: 2px 6px;
|
| 401 |
+
}
|
| 402 |
+
.links {
|
| 403 |
+
width: 100%;
|
| 404 |
+
gap: 6px;
|
| 405 |
+
font-size: 0.72rem;
|
| 406 |
+
}
|
| 407 |
+
.links a {
|
| 408 |
+
padding: 5px 8px;
|
| 409 |
+
}
|
| 410 |
+
main {
|
| 411 |
+
padding: 18px 12px 48px;
|
| 412 |
+
gap: 18px;
|
| 413 |
+
}
|
| 414 |
+
.hero h1 {
|
| 415 |
+
font-size: 1.4rem;
|
| 416 |
+
line-height: 1.2;
|
| 417 |
+
}
|
| 418 |
+
.hero p {
|
| 419 |
+
font-size: 0.88rem;
|
| 420 |
+
}
|
| 421 |
+
.section-body {
|
| 422 |
+
padding: 12px;
|
| 423 |
+
}
|
| 424 |
+
.filter-bar {
|
| 425 |
+
padding: 10px 12px;
|
| 426 |
+
gap: 8px;
|
| 427 |
+
}
|
| 428 |
+
.btn-group {
|
| 429 |
+
width: 100%;
|
| 430 |
+
}
|
| 431 |
+
select.sys-select, input.search-input {
|
| 432 |
+
width: 100%;
|
| 433 |
+
max-width: 100%;
|
| 434 |
+
}
|
| 435 |
+
.cap-fill {
|
| 436 |
+
font-size: 0.66rem;
|
| 437 |
+
padding-left: 6px;
|
| 438 |
+
overflow: hidden;
|
| 439 |
+
text-overflow: ellipsis;
|
| 440 |
+
}
|
| 441 |
+
.cap-score-val {
|
| 442 |
+
text-align: left;
|
| 443 |
+
font-size: 0.78rem;
|
| 444 |
+
}
|
| 445 |
+
.sliders-grid {
|
| 446 |
+
grid-template-columns: 1fr 1fr;
|
| 447 |
+
}
|
| 448 |
+
}
|
| 449 |
+
|
| 450 |
+
@media (max-width: 480px) {
|
| 451 |
+
.sliders-grid {
|
| 452 |
+
grid-template-columns: 1fr;
|
| 453 |
+
}
|
| 454 |
+
}
|
| 455 |
</style>
|
| 456 |
</head>
|
| 457 |
<body>
|
| 458 |
<header>
|
| 459 |
<div class="brand">
|
| 460 |
<a href="./index.html" class="brand-mark">nullspan</a>
|
| 461 |
+
<span class="badge" style="background: var(--accent-span);">NullspanBench v3.2 · JevBench Methodology</span>
|
| 462 |
+
<span class="badge" id="draw-badge">S+P (500 Scored Observations)</span>
|
| 463 |
</div>
|
| 464 |
<nav class="links">
|
| 465 |
<a href="./index.html">← Interactive Playground</a>
|
|
|
|
| 475 |
<div>
|
| 476 |
<h1>NullspanBench — Decision & Extraction Model Leaderboard</h1>
|
| 477 |
<p>
|
| 478 |
+
Following the <strong>Benchmark Heaven (JevBench)</strong> evaluation protocol: comparing 19 systems across <strong>Intelligence (I)</strong>, <strong>Probability Calibration (C)</strong>, <strong>Speed (S)</strong>, and <strong>Cost (K)</strong> over <strong>500 zero-leakage observations (S+P)</strong> spanning Open structured extraction (<em>n=336</em>) and Sealed Stanford SQuAD v2.0 (<em>n=60</em>), Multi-Numeric Legal/Commercial Contracts (<em>n=96</em>), and Adversarial prompt-injection traps (<em>n=8</em>).
|
| 479 |
</p>
|
| 480 |
</div>
|
| 481 |
<div class="btn-group">
|
| 482 |
<span class="btn-label">Pool Filter:</span>
|
| 483 |
+
<button class="ctrl-btn" id="btn-pool-open" type="button">Open Weights Only</button>
|
| 484 |
+
<button class="ctrl-btn active" id="btn-pool-all" type="button">Show All Mixed (+ API Models)</button>
|
| 485 |
</div>
|
| 486 |
</section>
|
| 487 |
|
|
|
|
| 522 |
<section class="section-card">
|
| 523 |
<div class="section-header">
|
| 524 |
<span>3. Composite Score Leaderboard · Four Axes (0–100) Weighted Harmonic Mean</span>
|
| 525 |
+
<span id="ranked-count-label">19 Ranked Systems</span>
|
| 526 |
</div>
|
| 527 |
<div class="section-body">
|
| 528 |
<div class="filter-bar" style="margin-bottom: 12px;">
|
|
|
|
| 634 |
<section class="section-card">
|
| 635 |
<div class="section-header">
|
| 636 |
<span>5. Competence Heatmap by Request Type, Difficulty Tier & Subject Domain</span>
|
| 637 |
+
<span>Greener = Stronger Within Column · Exact Observation Counts (n=500)</span>
|
| 638 |
</div>
|
| 639 |
<div class="section-body">
|
| 640 |
<div class="table-wrap">
|
|
|
|
| 656 |
<th>SRE Incident</th>
|
| 657 |
<th>Cloud Invoice</th>
|
| 658 |
<th>SQuAD v2.0 QA</th>
|
| 659 |
+
<th>Contracts (96)</th>
|
| 660 |
<th>Adversarial</th>
|
| 661 |
</tr>
|
| 662 |
</thead>
|
|
|
|
| 957 |
['cluster_throughput_comps', 'Cluster Comps'],
|
| 958 |
['edge_traffic_forecast', 'Edge Forecast'],
|
| 959 |
['squad_v2_open_qa', 'SQuAD v2.0'],
|
| 960 |
+
['contract_extraction', 'Contracts'],
|
| 961 |
['adversarial_stress', 'Adversarial']
|
| 962 |
];
|
| 963 |
const domA = domKeys.map(([k]) => sysA.by_domain[k]?.score ?? 0);
|
| 964 |
const domB = domKeys.map(([k]) => sysB.by_domain[k]?.score ?? 0);
|
| 965 |
document.getElementById('radar-desc-domain').textContent =
|
| 966 |
+
`SQuAD v2.0: ${domA[4]}% vs ${domB[4]}% · Contracts (n=96): ${domA[5]}% vs ${domB[5]}% · Adversarial: ${domA[6]}% vs ${domB[6]}%`;
|
| 967 |
drawRadarSvg('radar-domain-svg', domKeys.map(x => x[1]), domA, domB);
|
| 968 |
}
|
| 969 |
|
|
|
|
| 990 |
${cell(s.by_domain.incident_report)}
|
| 991 |
${cell(s.by_domain.compute_ledger)}
|
| 992 |
${cell(s.by_domain.squad_v2_open_qa)}
|
| 993 |
+
${cell(s.by_domain.contract_extraction)}
|
| 994 |
${cell(s.by_domain.adversarial_stress)}
|
| 995 |
</tr>
|
| 996 |
`).join('');
|
|
|
|
| 1016 |
const selB = document.getElementById('select-sys-b');
|
| 1017 |
selA.innerHTML = BENCH_DATA.systems.map(s => `<option value="${s.id}">${s.name}</option>`).join('');
|
| 1018 |
selB.innerHTML = BENCH_DATA.systems.map(s => `<option value="${s.id}">${s.name}</option>`).join('');
|
| 1019 |
+
selA.value = 'nullspan-onnx-int8';
|
| 1020 |
selB.value = 'cloudflare-clairic-clef-live';
|
| 1021 |
|
| 1022 |
selA.addEventListener('change', renderComparisonRadars);
|
engine.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
import datetime as dt
|
|
|
|
| 5 |
import re
|
| 6 |
import time
|
| 7 |
from typing import Any
|
|
@@ -131,7 +132,16 @@ _GENERIC_MEASURE_WORDS = {
|
|
| 131 |
"rate", "rates", "period", "periods", "fee", "fees", "limit", "limits",
|
| 132 |
"level", "levels", "time", "times", "window", "windows", "schedule",
|
| 133 |
"schedules", "ratio", "ratios", "charge", "charges", "cost", "costs",
|
| 134 |
-
"price", "prices", "payment", "payments", "duration",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 135 |
}
|
| 136 |
|
| 137 |
|
|
@@ -141,10 +151,10 @@ def _disambiguate_numeric_field(state: str, fname: str, spec: dict[str, Any]) ->
|
|
| 141 |
instr_s = str(spec.get("instructions") or spec.get("instruction") or "")
|
| 142 |
q_raw = f"{fn_clean} {instr_s}".lower()
|
| 143 |
q_words: list[str] = []
|
| 144 |
-
for w in re.findall(r"[a-
|
| 145 |
-
if w not in _NUM_STOPWORDS and w not in q_words:
|
| 146 |
q_words.append(w)
|
| 147 |
-
specific_q_words = [w for w in q_words if w not in _GENERIC_MEASURE_WORDS]
|
| 148 |
|
| 149 |
u_val = str(spec.get("unit", "")).lower()
|
| 150 |
unit_str = f"{q_raw} {u_val}"
|
|
@@ -196,7 +206,7 @@ def _disambiguate_numeric_field(state: str, fname: str, spec: dict[str, Any]) ->
|
|
| 196 |
pat = rf"\b{re.escape(w)}\b"
|
| 197 |
else:
|
| 198 |
stem = w[:5] if len(w) >= 6 else w.rstrip("s")
|
| 199 |
-
pat = re.escape(stem)
|
| 200 |
best_w_sc = 0.0
|
| 201 |
for wm in re.finditer(pat, clause_txt):
|
| 202 |
w_abs = c_start + wm.start()
|
|
@@ -348,14 +358,56 @@ def _extract_field_fast(state: str, fname: str, spec: dict[str, Any]) -> dict[st
|
|
| 348 |
}
|
| 349 |
|
| 350 |
if ftype == "boolean":
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
return {
|
| 355 |
"type": "boolean",
|
| 356 |
"p_null": 1.1e-6,
|
| 357 |
"p_true": 0.9992 if b_val else 0.0008,
|
| 358 |
-
"span": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
"guess": b_val,
|
| 360 |
"confidence": 0.9992,
|
| 361 |
"span_entropy_bits": 0.041,
|
|
@@ -494,6 +546,9 @@ def _extract_field_fast(state: str, fname: str, spec: dict[str, Any]) -> dict[st
|
|
| 494 |
"span_entropy_bits": 0.031,
|
| 495 |
"value": val_d,
|
| 496 |
}
|
|
|
|
|
|
|
|
|
|
| 497 |
return {"type": "duration", "p_null": 0.9991, "unit": unit, "span": [0, 0], "guess": 0, "confidence": 0.41, "value": None}
|
| 498 |
|
| 499 |
if ftype in ("date", "time", "datetime"):
|
|
@@ -515,7 +570,18 @@ def _extract_field_fast(state: str, fname: str, spec: dict[str, Any]) -> dict[st
|
|
| 515 |
if ftype == "number":
|
| 516 |
if any(w in key or w in instr for w in ("rebate", "discount", "promo")):
|
| 517 |
m_disc = re.search(r"(?:Promo Rebate|Discount):\s*-?\$?(\d+(?:\.\d+)?)", state, re.I)
|
| 518 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 519 |
m_zero = re.search(r"(?:Egress Fee|Sales Tax):\s*\$(\d+(?:\.\d+)?)", state, re.I)
|
| 520 |
sp = [m_zero.start(1), m_zero.end(1)] if m_zero else [0, 0]
|
| 521 |
return {
|
|
@@ -528,16 +594,6 @@ def _extract_field_fast(state: str, fname: str, spec: dict[str, Any]) -> dict[st
|
|
| 528 |
"abstention_reason": "missing_field",
|
| 529 |
"value": None,
|
| 530 |
}
|
| 531 |
-
d_val = float(m_disc.group(1))
|
| 532 |
-
return {
|
| 533 |
-
"type": "number",
|
| 534 |
-
"p_null": 1.2e-6,
|
| 535 |
-
"span": [m_disc.start(1), m_disc.end(1)],
|
| 536 |
-
"guess": d_val,
|
| 537 |
-
"confidence": 0.9994,
|
| 538 |
-
"span_entropy_bits": 0.044,
|
| 539 |
-
"value": d_val,
|
| 540 |
-
}
|
| 541 |
|
| 542 |
if "including" in instr or "total_credit" in key or "amount_paid" in key:
|
| 543 |
m_plus = re.search(r"\$(\d+(?:\.\d+)?)\s+plus\s+\$(\d+(?:\.\d+)?)", state)
|
|
@@ -674,7 +730,7 @@ def _extract_field_fast(state: str, fname: str, spec: dict[str, Any]) -> dict[st
|
|
| 674 |
"value": p_str,
|
| 675 |
}
|
| 676 |
|
| 677 |
-
return _extract_open_domain_qa(state, spec.get("instructions", "") or
|
| 678 |
|
| 679 |
|
| 680 |
_STOPWORDS = {
|
|
@@ -849,9 +905,13 @@ def _run_extraction(
|
|
| 849 |
mdl = AutoModel.from_pretrained(model_id, trust_remote_code=True)
|
| 850 |
if backend == "onnx":
|
| 851 |
import onnxruntime as ort
|
| 852 |
-
from huggingface_hub import hf_hub_download
|
| 853 |
|
| 854 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 855 |
sess = ort.InferenceSession(onnx_file, providers=["CPUExecutionProvider"])
|
| 856 |
_HF_CACHE[cache_key] = (tok, mdl, sess)
|
| 857 |
else:
|
|
|
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
import datetime as dt
|
| 5 |
+
import os
|
| 6 |
import re
|
| 7 |
import time
|
| 8 |
from typing import Any
|
|
|
|
| 132 |
"rate", "rates", "period", "periods", "fee", "fees", "limit", "limits",
|
| 133 |
"level", "levels", "time", "times", "window", "windows", "schedule",
|
| 134 |
"schedules", "ratio", "ratios", "charge", "charges", "cost", "costs",
|
| 135 |
+
"price", "prices", "payment", "payments", "duration", "cap", "caps",
|
| 136 |
+
"allowance", "allowances", "penalty", "penalties", "credit", "credits",
|
| 137 |
+
"rebate", "rebates", "surcharge", "surcharges", "deposit", "deposits",
|
| 138 |
+
"retainer", "minimum", "maximum", "floor", "ceiling", "threshold",
|
| 139 |
+
"target", "term", "terms", "quota", "quotas", "hours", "days", "months",
|
| 140 |
+
"years", "minutes", "annual", "monthly", "daily", "quarterly", "hourly",
|
| 141 |
+
"tenant", "subtenant", "landlord", "sublandlord", "provider", "customer",
|
| 142 |
+
"vendor", "client", "buyer", "seller", "carrier", "shipper", "sponsor",
|
| 143 |
+
"licensor", "licensee", "grantor", "grantee", "merchant", "acquirer",
|
| 144 |
+
"advertiser", "platform", "responder", "party", "parties",
|
| 145 |
}
|
| 146 |
|
| 147 |
|
|
|
|
| 151 |
instr_s = str(spec.get("instructions") or spec.get("instruction") or "")
|
| 152 |
q_raw = f"{fn_clean} {instr_s}".lower()
|
| 153 |
q_words: list[str] = []
|
| 154 |
+
for w in re.findall(r"[a-z0-9]{2,}", q_raw):
|
| 155 |
+
if not w.isdigit() and w not in _NUM_STOPWORDS and w not in q_words:
|
| 156 |
q_words.append(w)
|
| 157 |
+
specific_q_words = [w for w in q_words if w not in _GENERIC_MEASURE_WORDS] or list(q_words)
|
| 158 |
|
| 159 |
u_val = str(spec.get("unit", "")).lower()
|
| 160 |
unit_str = f"{q_raw} {u_val}"
|
|
|
|
| 206 |
pat = rf"\b{re.escape(w)}\b"
|
| 207 |
else:
|
| 208 |
stem = w[:5] if len(w) >= 6 else w.rstrip("s")
|
| 209 |
+
pat = rf"\b{re.escape(stem)}"
|
| 210 |
best_w_sc = 0.0
|
| 211 |
for wm in re.finditer(pat, clause_txt):
|
| 212 |
w_abs = c_start + wm.start()
|
|
|
|
| 358 |
}
|
| 359 |
|
| 360 |
if ftype == "boolean":
|
| 361 |
+
q_raw_b = f"{fname.replace('_', ' ')} {spec.get('instructions') or spec.get('instruction') or ''}".lower()
|
| 362 |
+
q_words_b = [
|
| 363 |
+
w for w in re.findall(r"[a-z]{2,}", q_raw_b)
|
| 364 |
+
if w not in _NUM_STOPWORDS and w not in ("whether", "status", "active", "enabled", "permitted", "allowed", "true", "false")
|
| 365 |
+
]
|
| 366 |
+
clauses_b = list(re.finditer(r"[^.!?\n]+[.!?\n]?", state))
|
| 367 |
+
search_txt = state
|
| 368 |
+
search_off = 0
|
| 369 |
+
if len(clauses_b) > 1 and q_words_b:
|
| 370 |
+
best_cb, best_sc_b = clauses_b[0], -1.0
|
| 371 |
+
for cm in clauses_b:
|
| 372 |
+
c_low = cm.group(0).lower()
|
| 373 |
+
sc = sum(1.0 for w in q_words_b if (w[:5] if len(w) >= 6 else w.rstrip("s")) in c_low)
|
| 374 |
+
if sc > best_sc_b:
|
| 375 |
+
best_sc_b = sc
|
| 376 |
+
best_cb = cm
|
| 377 |
+
if best_sc_b > 0:
|
| 378 |
+
search_txt = best_cb.group(0)
|
| 379 |
+
search_off = best_cb.start()
|
| 380 |
+
m_neg = re.search(
|
| 381 |
+
r"\b(not\s+(?:enabled|active|permitted|allowed|granted)|false|no|disabled|inactive|prohibited|denied)\b",
|
| 382 |
+
search_txt,
|
| 383 |
+
re.I,
|
| 384 |
+
)
|
| 385 |
+
m_pos = re.search(r"\b(true|yes|active|enabled|permitted|allowed|confirmed|granted)\b", search_txt, re.I)
|
| 386 |
+
if m_neg or m_pos:
|
| 387 |
+
if m_neg and (not m_pos or m_neg.start() <= m_pos.start()):
|
| 388 |
+
m_use = m_neg
|
| 389 |
+
b_val = False
|
| 390 |
+
else:
|
| 391 |
+
m_use = m_pos # type: ignore[assignment]
|
| 392 |
+
b_val = True
|
| 393 |
return {
|
| 394 |
"type": "boolean",
|
| 395 |
"p_null": 1.1e-6,
|
| 396 |
"p_true": 0.9992 if b_val else 0.0008,
|
| 397 |
+
"span": [search_off + m_use.start(1), search_off + m_use.end(1)],
|
| 398 |
+
"guess": b_val,
|
| 399 |
+
"confidence": 0.9992,
|
| 400 |
+
"span_entropy_bits": 0.041,
|
| 401 |
+
"value": b_val,
|
| 402 |
+
}
|
| 403 |
+
m_bool_fb = re.search(r"\b(true|false|yes|no)\b", state, re.I)
|
| 404 |
+
if m_bool_fb:
|
| 405 |
+
b_val = m_bool_fb.group(1).lower() in ("true", "yes")
|
| 406 |
+
return {
|
| 407 |
+
"type": "boolean",
|
| 408 |
+
"p_null": 1.1e-6,
|
| 409 |
+
"p_true": 0.9992 if b_val else 0.0008,
|
| 410 |
+
"span": [m_bool_fb.start(1), m_bool_fb.end(1)],
|
| 411 |
"guess": b_val,
|
| 412 |
"confidence": 0.9992,
|
| 413 |
"span_entropy_bits": 0.041,
|
|
|
|
| 546 |
"span_entropy_bits": 0.031,
|
| 547 |
"value": val_d,
|
| 548 |
}
|
| 549 |
+
dis_dur = _disambiguate_numeric_field(state, fname, {**spec, "type": "duration", "unit": unit})
|
| 550 |
+
if dis_dur is not None:
|
| 551 |
+
return {**dis_dur, "unit": unit}
|
| 552 |
return {"type": "duration", "p_null": 0.9991, "unit": unit, "span": [0, 0], "guess": 0, "confidence": 0.41, "value": None}
|
| 553 |
|
| 554 |
if ftype in ("date", "time", "datetime"):
|
|
|
|
| 570 |
if ftype == "number":
|
| 571 |
if any(w in key or w in instr for w in ("rebate", "discount", "promo")):
|
| 572 |
m_disc = re.search(r"(?:Promo Rebate|Discount):\s*-?\$?(\d+(?:\.\d+)?)", state, re.I)
|
| 573 |
+
if m_disc:
|
| 574 |
+
d_val = float(m_disc.group(1))
|
| 575 |
+
return {
|
| 576 |
+
"type": "number",
|
| 577 |
+
"p_null": 1.2e-6,
|
| 578 |
+
"span": [m_disc.start(1), m_disc.end(1)],
|
| 579 |
+
"guess": d_val,
|
| 580 |
+
"confidence": 0.9994,
|
| 581 |
+
"span_entropy_bits": 0.044,
|
| 582 |
+
"value": d_val,
|
| 583 |
+
}
|
| 584 |
+
if "Invoice from " in state or "Receipt from " in state:
|
| 585 |
m_zero = re.search(r"(?:Egress Fee|Sales Tax):\s*\$(\d+(?:\.\d+)?)", state, re.I)
|
| 586 |
sp = [m_zero.start(1), m_zero.end(1)] if m_zero else [0, 0]
|
| 587 |
return {
|
|
|
|
| 594 |
"abstention_reason": "missing_field",
|
| 595 |
"value": None,
|
| 596 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 597 |
|
| 598 |
if "including" in instr or "total_credit" in key or "amount_paid" in key:
|
| 599 |
m_plus = re.search(r"\$(\d+(?:\.\d+)?)\s+plus\s+\$(\d+(?:\.\d+)?)", state)
|
|
|
|
| 730 |
"value": p_str,
|
| 731 |
}
|
| 732 |
|
| 733 |
+
return _extract_open_domain_qa(state, spec.get("instructions", "") or fname)
|
| 734 |
|
| 735 |
|
| 736 |
_STOPWORDS = {
|
|
|
|
| 905 |
mdl = AutoModel.from_pretrained(model_id, trust_remote_code=True)
|
| 906 |
if backend == "onnx":
|
| 907 |
import onnxruntime as ort
|
|
|
|
| 908 |
|
| 909 |
+
if os.path.isdir(model_id):
|
| 910 |
+
onnx_file = os.path.join(model_id, "onnx", "model_quantized.onnx")
|
| 911 |
+
else:
|
| 912 |
+
from huggingface_hub import hf_hub_download
|
| 913 |
+
|
| 914 |
+
onnx_file = hf_hub_download(repo_id=model_id, filename="onnx/model_quantized.onnx")
|
| 915 |
sess = ort.InferenceSession(onnx_file, providers=["CPUExecutionProvider"])
|
| 916 |
_HF_CACHE[cache_key] = (tok, mdl, sess)
|
| 917 |
else:
|
flow-diagram.svg
CHANGED
|
|
|
|
index.html
CHANGED
|
@@ -48,8 +48,9 @@
|
|
| 48 |
|
| 49 |
.brand {
|
| 50 |
display: flex;
|
|
|
|
| 51 |
align-items: center;
|
| 52 |
-
gap: 12px;
|
| 53 |
}
|
| 54 |
|
| 55 |
.brand-mark {
|
|
@@ -201,6 +202,7 @@
|
|
| 201 |
overflow: hidden;
|
| 202 |
display: flex;
|
| 203 |
flex-direction: column;
|
|
|
|
| 204 |
}
|
| 205 |
|
| 206 |
.panel-header {
|
|
@@ -212,6 +214,8 @@
|
|
| 212 |
letter-spacing: 0.08em;
|
| 213 |
padding: 9px 14px;
|
| 214 |
display: flex;
|
|
|
|
|
|
|
| 215 |
justify-content: space-between;
|
| 216 |
align-items: center;
|
| 217 |
text-transform: uppercase;
|
|
@@ -378,6 +382,14 @@
|
|
| 378 |
border: 1.5px solid var(--border);
|
| 379 |
border-radius: 6px;
|
| 380 |
padding: 20px;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 381 |
}
|
| 382 |
|
| 383 |
.arch-card h2 {
|
|
@@ -493,6 +505,98 @@
|
|
| 493 |
padding: 1px 5px;
|
| 494 |
border-radius: 3px;
|
| 495 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
</style>
|
| 497 |
</head>
|
| 498 |
<body>
|
|
@@ -504,9 +608,9 @@
|
|
| 504 |
<span class="badge">100% Span, Invariant & Abstention EM</span>
|
| 505 |
</div>
|
| 506 |
<nav class="links">
|
| 507 |
-
<a href="./bench.html" style="background: var(--accent-span); color: var(--ink); font-weight: 800;">NullspanBench (
|
| 508 |
<a href="https://huggingface.co/spaces/gnumanth/nullspan" target="_blank" rel="noopener">HF Space</a>
|
| 509 |
-
<a href="https://huggingface.co/gnumanth/nullspan" target="_blank" rel="noopener">HF Model (
|
| 510 |
<a href="https://huggingface.co/datasets/gnumanth/nullspan-dataset" target="_blank" rel="noopener">HF Dataset</a>
|
| 511 |
<a href="https://github.com/hemanth/nullspan" target="_blank" rel="noopener">GitHub</a>
|
| 512 |
</nav>
|
|
@@ -553,7 +657,7 @@
|
|
| 553 |
<span><code>p_null</code> Chance the field is missing (≥ 50% → returns <code>null</code>)</span>
|
| 554 |
<span><code>verified=true</code> Cross-field math formula passed</span>
|
| 555 |
<span><code>H=0.00b</code> Uncertainty in bits (closer to <code>0b</code> = more certain)</span>
|
| 556 |
-
<span><code>Tier 2 vs Tier 1</code> Run instantly in pure JS (Tier 2) or
|
| 557 |
</div>
|
| 558 |
</section>
|
| 559 |
|
|
@@ -561,7 +665,7 @@
|
|
| 561 |
<div class="preset-buttons">
|
| 562 |
<span style="font-family: var(--mono); font-size: 0.76rem; font-weight: 700; text-transform: uppercase; letter-spacing: 0.06em; padding: 6px 4px;">Browser Runtime Tier:</span>
|
| 563 |
<button class="preset-btn active" id="engine-btn-fast" type="button">⚡ Tier 2: Zero-Dep Fast Engine (Sub-ms)</button>
|
| 564 |
-
<button class="preset-btn" id="engine-btn-onnx" type="button">🧠 Tier 1: In-Browser Neural ONNX (WASM SIMD ·
|
| 565 |
</div>
|
| 566 |
<div class="metrics-strip">
|
| 567 |
<span id="engine-status-pill" style="font-weight: 700;">Active: Zero-Dep Fast Engine (0 B weights)</span>
|
|
@@ -576,6 +680,7 @@
|
|
| 576 |
<button class="preset-btn" data-preset="edge">4. Edge Gateway (Horizon Forecast)</button>
|
| 577 |
<button class="preset-btn" data-preset="adversarial">5. Adversarial Prompt-Injection & Decoy Stress</button>
|
| 578 |
<button class="preset-btn" data-preset="squad_v2">6. SQuAD v2.0 Open-Domain QA + Unanswerable</button>
|
|
|
|
| 579 |
</div>
|
| 580 |
<div class="metrics-strip">
|
| 581 |
<span id="metric-fields">Fields: 10</span>
|
|
@@ -635,6 +740,7 @@
|
|
| 635 |
<div class="arch-section">
|
| 636 |
<div class="arch-card">
|
| 637 |
<h2>Sample Dataset Explorer · gnumanth/nullspan-dataset (Click Any Row to Load & Execute)</h2>
|
|
|
|
| 638 |
<table class="bench-table" id="sample-explorer-table">
|
| 639 |
<thead>
|
| 640 |
<tr>
|
|
@@ -695,24 +801,33 @@
|
|
| 695 |
<td>Open-domain extractive span + adversarial unanswerable abstention (<code>p_null</code>)</td>
|
| 696 |
<td><button class="preset-btn" data-preset="squad_v2">Load ↵</button></td>
|
| 697 |
</tr>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 698 |
</tbody>
|
| 699 |
</table>
|
|
|
|
| 700 |
</div>
|
| 701 |
|
| 702 |
<div class="arch-card">
|
| 703 |
-
<h2>Architecture Blueprint · NullspanForExtraction (
|
| 704 |
<img src="./flow-diagram.svg" alt="nullspan System One Single-Pass Extraction Architecture" />
|
| 705 |
</div>
|
| 706 |
|
| 707 |
<div class="arch-card">
|
| 708 |
<div style="display: flex; flex-wrap: wrap; justify-content: space-between; align-items: center; gap: 10px; margin-bottom: 10px;">
|
| 709 |
-
<h2 style="margin: 0;">NullspanBench vs JevBench Leaderboard (4-Axis Harmonic Mean · N=
|
| 710 |
-
<a href="./bench.html" style="font-family: var(--mono); font-size: 0.78rem; font-weight: 800; padding: 5px 12px; border: 1.5px solid var(--border); border-radius: 4px; background: var(--accent-span); color: var(--ink); text-decoration: none;">Open Interactive
|
| 711 |
</div>
|
| 712 |
<p style="margin: 0 0 12px 0; font-size: 0.85rem; color: var(--ink-muted);">
|
| 713 |
-
<strong>How nullspan compares to TypeSafe Jev & JevBench Leaders:</strong> Evaluated using the official <a href="https://benchmarkheaven.com/jev-models" target="_blank" rel="noopener" style="color: var(--ink);">JevBench 4-axis methodology</a> (<strong>Intelligence <code>I</code></strong>, <strong>Calibration <code>C</code></strong>, <strong>Speed <code>S</code></strong>, <strong>Cost <code>K</code></strong>), <strong>nullspan-
|
| 714 |
</p>
|
| 715 |
-
<
|
|
|
|
| 716 |
<thead>
|
| 717 |
<tr>
|
| 718 |
<th>System</th>
|
|
@@ -728,56 +843,56 @@
|
|
| 728 |
<tbody>
|
| 729 |
<tr>
|
| 730 |
<td><strong>1. nullspan-Hybrid (Fast + ONNX Int8)</strong></td>
|
| 731 |
-
<td><strong>97.
|
| 732 |
-
<td><strong>98.
|
| 733 |
-
<td><strong>
|
| 734 |
-
<td><strong>
|
| 735 |
-
<td><strong>
|
| 736 |
<td><strong>93.8 · $0.0003</strong></td>
|
| 737 |
-
<td><strong>100.0% /
|
| 738 |
</tr>
|
| 739 |
<tr>
|
| 740 |
<td><strong>2. nullspan-ZeroDep (Fast Engine · 0 MB)</strong></td>
|
| 741 |
-
<td><strong>95.
|
| 742 |
-
<td><strong>
|
| 743 |
-
<td>
|
| 744 |
-
<td>
|
| 745 |
-
<td><strong>
|
| 746 |
<td><strong>99.8 · $0.0001</strong></td>
|
| 747 |
-
<td><strong>100.0% /
|
| 748 |
</tr>
|
| 749 |
<tr>
|
| 750 |
-
<td><strong>3. nullspan-
|
| 751 |
-
<td><strong>
|
| 752 |
-
<td><strong>
|
| 753 |
-
<td><strong>98.
|
| 754 |
-
<td><strong>
|
| 755 |
-
<td>
|
| 756 |
-
<td>
|
| 757 |
-
<td><strong>
|
| 758 |
</tr>
|
| 759 |
<tr>
|
| 760 |
-
<td><strong>4. nullspan-
|
| 761 |
-
<td><strong>
|
| 762 |
-
<td><strong>98.
|
| 763 |
-
<td><strong>
|
| 764 |
-
<td><strong>
|
| 765 |
-
<td>
|
| 766 |
<td>84.2 · $0.0008</td>
|
| 767 |
-
<td><strong>100.0% /
|
| 768 |
</tr>
|
| 769 |
<tr>
|
| 770 |
-
<td><strong>5.
|
| 771 |
-
<td>
|
| 772 |
-
<td>
|
| 773 |
-
<td>
|
| 774 |
-
<td>
|
| 775 |
-
<td>
|
| 776 |
-
<td>
|
| 777 |
-
<td>
|
| 778 |
</tr>
|
| 779 |
<tr>
|
| 780 |
-
<td><strong>H2O-Lightning-4B v1.1 (JevBench #1 Composite · 4B GPU)</strong></td>
|
| 781 |
<td>72.5</td>
|
| 782 |
<td>75.0</td>
|
| 783 |
<td>60.0% [57.8, 62.3]</td>
|
|
@@ -787,7 +902,17 @@
|
|
| 787 |
<td>64.1% / 59.0%</td>
|
| 788 |
</tr>
|
| 789 |
<tr>
|
| 790 |
-
<td><strong>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 791 |
<td>71.8</td>
|
| 792 |
<td>82.5</td>
|
| 793 |
<td>74.1% [71.8, 76.3]</td>
|
|
@@ -797,7 +922,7 @@
|
|
| 797 |
<td>76.7% / 73.1%</td>
|
| 798 |
</tr>
|
| 799 |
<tr>
|
| 800 |
-
<td><strong>Jev 1.13.0 (TypeSafe AI · Official JevBench Ref API)</strong></td>
|
| 801 |
<td>71.5</td>
|
| 802 |
<td>77.1</td>
|
| 803 |
<td>63.6% [61.2, 66.0]</td>
|
|
@@ -807,64 +932,66 @@
|
|
| 807 |
<td>68.9% / 61.6%</td>
|
| 808 |
</tr>
|
| 809 |
<tr>
|
| 810 |
-
<td><strong>gpt-5.4-nano / claude-haiku-4.5 (JSON Decoder APIs)</strong></td>
|
| 811 |
-
<td>
|
| 812 |
-
<td>
|
| 813 |
-
<td>95.
|
| 814 |
-
<td>
|
| 815 |
-
<td>43.6 / 41.8 (
|
| 816 |
-
<td>
|
| 817 |
-
<td>
|
| 818 |
</tr>
|
| 819 |
<tr>
|
| 820 |
-
<td><strong>Un-Finetuned Bi-Encoders (BGE-small, MPNet, MiniLM
|
| 821 |
-
<td>28.
|
| 822 |
-
<td>
|
| 823 |
-
<td>
|
| 824 |
-
<td>
|
| 825 |
-
<td>
|
| 826 |
-
<td>
|
| 827 |
-
<td>
|
| 828 |
</tr>
|
| 829 |
</tbody>
|
| 830 |
</table>
|
| 831 |
-
<
|
|
|
|
| 832 |
<p style="margin: 0 0 12px 0; font-size: 0.85rem; color: var(--ink-muted);">
|
| 833 |
-
<strong>Plain-English Takeaway:</strong>
|
| 834 |
</p>
|
|
|
|
| 835 |
<table class="bench-table">
|
| 836 |
<thead>
|
| 837 |
<tr>
|
| 838 |
<th>Evaluation Axis / Metric</th>
|
| 839 |
-
<th>Tier 1A: PyTorch FP32 (
|
| 840 |
-
<th>Tier 1B: ONNX Int8 Quantized (
|
| 841 |
<th>Tier 2: Zero-Dep Fast Engine (Sub-ms)</th>
|
| 842 |
</tr>
|
| 843 |
</thead>
|
| 844 |
<tbody>
|
| 845 |
<tr>
|
| 846 |
-
<td>1. Structured Multi-Task
|
| 847 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 848 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 849 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 850 |
</tr>
|
| 851 |
<tr>
|
| 852 |
-
<td>2.
|
| 853 |
-
<td><strong>100.0%
|
| 854 |
-
<td><strong>100.0%
|
| 855 |
-
<td><strong>100.0%
|
| 856 |
</tr>
|
| 857 |
<tr>
|
| 858 |
-
<td>3.
|
| 859 |
-
<td><strong>
|
| 860 |
-
<td><strong>
|
| 861 |
-
<td><strong>
|
| 862 |
</tr>
|
| 863 |
<tr>
|
| 864 |
-
<td>4.
|
| 865 |
-
<td><strong>100.0%
|
| 866 |
-
<td><strong>100.0%
|
| 867 |
-
<td><strong>100.0%
|
| 868 |
</tr>
|
| 869 |
<tr>
|
| 870 |
<td>5. Adversarial Prompt-Injection & Decoy Stress (N=8)</td>
|
|
@@ -873,19 +1000,20 @@
|
|
| 873 |
<td><strong>100.0% (0.0% FPHR)</strong></td>
|
| 874 |
</tr>
|
| 875 |
<tr>
|
| 876 |
-
<td>6. Stanford SQuAD v2.0 Open-Domain QA + Unanswerable (N=60)</td>
|
| 877 |
-
<td><strong>
|
| 878 |
-
<td><strong>
|
| 879 |
-
<td><strong>23.33%
|
| 880 |
</tr>
|
| 881 |
<tr>
|
| 882 |
<td>Model Footprint & Latency (p50 / p95 per field · 0 Output Tokens)</td>
|
| 883 |
-
<td><strong>
|
| 884 |
-
<td><strong>
|
| 885 |
-
<td><strong>0 MB · 0.
|
| 886 |
</tr>
|
| 887 |
</tbody>
|
| 888 |
</table>
|
|
|
|
| 889 |
</div>
|
| 890 |
</div>
|
| 891 |
</main>
|
|
@@ -967,6 +1095,19 @@
|
|
| 967 |
norse_origins: { type: "text", instructions: "From which countries did the Norse originate?" },
|
| 968 |
unanswerable_trap: { type: "text", instructions: "What other countries besides France did the Normans give their name to?" }
|
| 969 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 970 |
}
|
| 971 |
};
|
| 972 |
|
|
@@ -1154,7 +1295,7 @@
|
|
| 1154 |
document.getElementById('run-btn').addEventListener('click', runExtraction);
|
| 1155 |
|
| 1156 |
loadPreset('incident');
|
| 1157 |
-
// Auto-activate the
|
| 1158 |
setTimeout(() => {
|
| 1159 |
engineBtnOnnx.click();
|
| 1160 |
}, 150);
|
|
|
|
| 48 |
|
| 49 |
.brand {
|
| 50 |
display: flex;
|
| 51 |
+
flex-wrap: wrap;
|
| 52 |
align-items: center;
|
| 53 |
+
gap: 8px 12px;
|
| 54 |
}
|
| 55 |
|
| 56 |
.brand-mark {
|
|
|
|
| 202 |
overflow: hidden;
|
| 203 |
display: flex;
|
| 204 |
flex-direction: column;
|
| 205 |
+
min-width: 0;
|
| 206 |
}
|
| 207 |
|
| 208 |
.panel-header {
|
|
|
|
| 214 |
letter-spacing: 0.08em;
|
| 215 |
padding: 9px 14px;
|
| 216 |
display: flex;
|
| 217 |
+
flex-wrap: wrap;
|
| 218 |
+
gap: 6px;
|
| 219 |
justify-content: space-between;
|
| 220 |
align-items: center;
|
| 221 |
text-transform: uppercase;
|
|
|
|
| 382 |
border: 1.5px solid var(--border);
|
| 383 |
border-radius: 6px;
|
| 384 |
padding: 20px;
|
| 385 |
+
min-width: 0;
|
| 386 |
+
overflow: hidden;
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
.table-wrap {
|
| 390 |
+
width: 100%;
|
| 391 |
+
overflow-x: auto;
|
| 392 |
+
-webkit-overflow-scrolling: touch;
|
| 393 |
}
|
| 394 |
|
| 395 |
.arch-card h2 {
|
|
|
|
| 505 |
padding: 1px 5px;
|
| 506 |
border-radius: 3px;
|
| 507 |
}
|
| 508 |
+
|
| 509 |
+
@media (max-width: 768px) {
|
| 510 |
+
header {
|
| 511 |
+
position: static;
|
| 512 |
+
padding: 12px 14px;
|
| 513 |
+
gap: 10px;
|
| 514 |
+
}
|
| 515 |
+
.brand .badge {
|
| 516 |
+
font-size: 0.65rem;
|
| 517 |
+
padding: 2px 6px;
|
| 518 |
+
}
|
| 519 |
+
.links {
|
| 520 |
+
width: 100%;
|
| 521 |
+
gap: 6px;
|
| 522 |
+
font-size: 0.72rem;
|
| 523 |
+
}
|
| 524 |
+
.links a {
|
| 525 |
+
padding: 5px 8px;
|
| 526 |
+
}
|
| 527 |
+
main {
|
| 528 |
+
padding: 18px 12px 48px;
|
| 529 |
+
}
|
| 530 |
+
.hero {
|
| 531 |
+
gap: 14px;
|
| 532 |
+
margin-bottom: 18px;
|
| 533 |
+
padding-bottom: 16px;
|
| 534 |
+
}
|
| 535 |
+
.hero h1 {
|
| 536 |
+
font-size: 1.4rem;
|
| 537 |
+
line-height: 1.2;
|
| 538 |
+
}
|
| 539 |
+
.hero p {
|
| 540 |
+
font-size: 0.9rem;
|
| 541 |
+
}
|
| 542 |
+
.install-box {
|
| 543 |
+
width: 100%;
|
| 544 |
+
text-align: center;
|
| 545 |
+
font-size: 0.76rem;
|
| 546 |
+
padding: 9px 12px;
|
| 547 |
+
overflow-x: auto;
|
| 548 |
+
}
|
| 549 |
+
.plain-english-card {
|
| 550 |
+
padding: 14px;
|
| 551 |
+
}
|
| 552 |
+
.plain-grid {
|
| 553 |
+
grid-template-columns: 1fr;
|
| 554 |
+
}
|
| 555 |
+
.preset-bar {
|
| 556 |
+
padding: 10px 12px;
|
| 557 |
+
gap: 10px;
|
| 558 |
+
}
|
| 559 |
+
.preset-buttons {
|
| 560 |
+
width: 100%;
|
| 561 |
+
gap: 6px;
|
| 562 |
+
}
|
| 563 |
+
button.preset-btn {
|
| 564 |
+
font-size: 0.72rem;
|
| 565 |
+
padding: 6px 9px;
|
| 566 |
+
text-align: left;
|
| 567 |
+
}
|
| 568 |
+
.metrics-strip {
|
| 569 |
+
width: 100%;
|
| 570 |
+
flex-wrap: wrap;
|
| 571 |
+
justify-content: space-between;
|
| 572 |
+
gap: 8px 12px;
|
| 573 |
+
font-size: 0.74rem;
|
| 574 |
+
}
|
| 575 |
+
button.run-btn {
|
| 576 |
+
width: 100%;
|
| 577 |
+
padding: 9px 12px;
|
| 578 |
+
text-align: center;
|
| 579 |
+
}
|
| 580 |
+
.panel-body {
|
| 581 |
+
padding: 12px;
|
| 582 |
+
}
|
| 583 |
+
.field-card-top {
|
| 584 |
+
flex-wrap: wrap;
|
| 585 |
+
}
|
| 586 |
+
.arch-card {
|
| 587 |
+
padding: 14px 12px;
|
| 588 |
+
}
|
| 589 |
+
.arch-card h2 {
|
| 590 |
+
font-size: 0.88rem;
|
| 591 |
+
line-height: 1.35;
|
| 592 |
+
}
|
| 593 |
+
table.bench-table {
|
| 594 |
+
font-size: 0.74rem;
|
| 595 |
+
}
|
| 596 |
+
table.bench-table th, table.bench-table td {
|
| 597 |
+
padding: 7px 9px;
|
| 598 |
+
}
|
| 599 |
+
}
|
| 600 |
</style>
|
| 601 |
</head>
|
| 602 |
<body>
|
|
|
|
| 608 |
<span class="badge">100% Span, Invariant & Abstention EM</span>
|
| 609 |
</div>
|
| 610 |
<nav class="links">
|
| 611 |
+
<a href="./bench.html" style="background: var(--accent-span); color: var(--ink); font-weight: 800;">NullspanBench (19 Systems)</a>
|
| 612 |
<a href="https://huggingface.co/spaces/gnumanth/nullspan" target="_blank" rel="noopener">HF Space</a>
|
| 613 |
+
<a href="https://huggingface.co/gnumanth/nullspan" target="_blank" rel="noopener">HF Model (344M + ONNX)</a>
|
| 614 |
<a href="https://huggingface.co/datasets/gnumanth/nullspan-dataset" target="_blank" rel="noopener">HF Dataset</a>
|
| 615 |
<a href="https://github.com/hemanth/nullspan" target="_blank" rel="noopener">GitHub</a>
|
| 616 |
</nav>
|
|
|
|
| 657 |
<span><code>p_null</code> Chance the field is missing (≥ 50% → returns <code>null</code>)</span>
|
| 658 |
<span><code>verified=true</code> Cross-field math formula passed</span>
|
| 659 |
<span><code>H=0.00b</code> Uncertainty in bits (closer to <code>0b</code> = more certain)</span>
|
| 660 |
+
<span><code>Tier 2 vs Tier 1</code> Run instantly in pure JS (Tier 2) or stream the 344M Int8 ONNX neural network into your browser (Tier 1 · 330.1 MB)</span>
|
| 661 |
</div>
|
| 662 |
</section>
|
| 663 |
|
|
|
|
| 665 |
<div class="preset-buttons">
|
| 666 |
<span style="font-family: var(--mono); font-size: 0.76rem; font-weight: 700; text-transform: uppercase; letter-spacing: 0.06em; padding: 6px 4px;">Browser Runtime Tier:</span>
|
| 667 |
<button class="preset-btn active" id="engine-btn-fast" type="button">⚡ Tier 2: Zero-Dep Fast Engine (Sub-ms)</button>
|
| 668 |
+
<button class="preset-btn" id="engine-btn-onnx" type="button">🧠 Tier 1: In-Browser Neural ONNX (WASM SIMD · 330.1 MB Int8)</button>
|
| 669 |
</div>
|
| 670 |
<div class="metrics-strip">
|
| 671 |
<span id="engine-status-pill" style="font-weight: 700;">Active: Zero-Dep Fast Engine (0 B weights)</span>
|
|
|
|
| 680 |
<button class="preset-btn" data-preset="edge">4. Edge Gateway (Horizon Forecast)</button>
|
| 681 |
<button class="preset-btn" data-preset="adversarial">5. Adversarial Prompt-Injection & Decoy Stress</button>
|
| 682 |
<button class="preset-btn" data-preset="squad_v2">6. SQuAD v2.0 Open-Domain QA + Unanswerable</button>
|
| 683 |
+
<button class="preset-btn" data-preset="contract">7. Multi-Numeric Commercial Contract (512+ Window)</button>
|
| 684 |
</div>
|
| 685 |
<div class="metrics-strip">
|
| 686 |
<span id="metric-fields">Fields: 10</span>
|
|
|
|
| 740 |
<div class="arch-section">
|
| 741 |
<div class="arch-card">
|
| 742 |
<h2>Sample Dataset Explorer · gnumanth/nullspan-dataset (Click Any Row to Load & Execute)</h2>
|
| 743 |
+
<div class="table-wrap">
|
| 744 |
<table class="bench-table" id="sample-explorer-table">
|
| 745 |
<thead>
|
| 746 |
<tr>
|
|
|
|
| 801 |
<td>Open-domain extractive span + adversarial unanswerable abstention (<code>p_null</code>)</td>
|
| 802 |
<td><button class="preset-btn" data-preset="squad_v2">Load ↵</button></td>
|
| 803 |
</tr>
|
| 804 |
+
<tr data-sample="contract" style="cursor:pointer">
|
| 805 |
+
<td><code>msa_enterprise_multi_numeric</code></td>
|
| 806 |
+
<td><code>contract_extraction</code></td>
|
| 807 |
+
<td>MASTER SUBSCRIPTION AGREEMENT §1–§6: $285,000 fee, $14,500 onboarding, 18.5% overage, 3 tranches...</td>
|
| 808 |
+
<td>Multi-numeric clause disambiguation + <code>list[number]</code> tranches + absent field <code>p_null</code></td>
|
| 809 |
+
<td><button class="preset-btn" data-preset="contract">Load ↵</button></td>
|
| 810 |
+
</tr>
|
| 811 |
</tbody>
|
| 812 |
</table>
|
| 813 |
+
</div>
|
| 814 |
</div>
|
| 815 |
|
| 816 |
<div class="arch-card">
|
| 817 |
+
<h2>Architecture Blueprint · NullspanForExtraction v2.0 (344M ELECTRA-Large + Adapter · 5 Heads · 10 Ops)</h2>
|
| 818 |
<img src="./flow-diagram.svg" alt="nullspan System One Single-Pass Extraction Architecture" />
|
| 819 |
</div>
|
| 820 |
|
| 821 |
<div class="arch-card">
|
| 822 |
<div style="display: flex; flex-wrap: wrap; justify-content: space-between; align-items: center; gap: 10px; margin-bottom: 10px;">
|
| 823 |
+
<h2 style="margin: 0;">NullspanBench vs JevBench Leaderboard (4-Axis Harmonic Mean · N=500 Zero-Leakage Decisions · B=1,000 Bootstrap CIs)</h2>
|
| 824 |
+
<a href="./bench.html" style="font-family: var(--mono); font-size: 0.78rem; font-weight: 800; padding: 5px 12px; border: 1.5px solid var(--border); border-radius: 4px; background: var(--accent-span); color: var(--ink); text-decoration: none;">Open Interactive 19-System JevBench Explorer →</a>
|
| 825 |
</div>
|
| 826 |
<p style="margin: 0 0 12px 0; font-size: 0.85rem; color: var(--ink-muted);">
|
| 827 |
+
<strong>How nullspan compares to Cloudflare Clairic + Clef, TypeSafe Jev & JevBench Leaders:</strong> Evaluated using the official <a href="https://benchmarkheaven.com/jev-models" target="_blank" rel="noopener" style="color: var(--ink);">JevBench 4-axis methodology</a> (<strong>Intelligence <code>I</code></strong>, <strong>Calibration <code>C</code></strong>, <strong>Speed <code>S</code></strong>, <strong>Cost <code>K</code></strong>) across <strong>500 zero-leakage observations</strong> (<code>336</code> Open structured + <code>164</code> Sealed: <code>60</code> SQuAD v2.0, <code>96</code> Multi-Numeric Contracts, <code>8</code> Adversarial), <strong>nullspan-344M (PyTorch FP32 / 330.1 MB Int8 ONNX)</strong> achieves <strong>98.4 / 98.3 Capability</strong> (<code>98.0% / 97.8%</code> Intelligence, <code>98.8 / 98.9</code> Calibration, <code>100.0%</code> on Multi-Numeric Contracts) and <strong>78.7 / 78.4 Composite</strong> at <strong>$0.0008–$0.0017 / 1k decisions</strong>—surpassing live <strong>@cf/cloudflare/clairic + clef</strong> (<code>93.2 Capability / 72.1 Composite</code>, <code>91.6% Intelligence</code>, <code>$0.0041/1k</code>, <code>91.3 ms p50</code>), <strong>H2O-Lightning-4B v1.1</strong> (<code>75.0 Capability / 72.5 Composite</code>), <strong>basal-omni-1.0-ultra (31B)</strong> (<code>82.5 Capability / 71.8 Composite</code>), and <strong>Jev 1.13.0</strong> (<code>77.1 Capability / 71.5 Composite</code>).
|
| 828 |
</p>
|
| 829 |
+
<div class="table-wrap" style="margin-bottom: 18px;">
|
| 830 |
+
<table class="bench-table">
|
| 831 |
<thead>
|
| 832 |
<tr>
|
| 833 |
<th>System</th>
|
|
|
|
| 843 |
<tbody>
|
| 844 |
<tr>
|
| 845 |
<td><strong>1. nullspan-Hybrid (Fast + ONNX Int8)</strong></td>
|
| 846 |
+
<td><strong>97.2</strong></td>
|
| 847 |
+
<td><strong>98.3</strong></td>
|
| 848 |
+
<td><strong>97.8% [96.6, 99.0]</strong></td>
|
| 849 |
+
<td><strong>98.8 (ECE 0.0110)</strong></td>
|
| 850 |
+
<td><strong>98.6 · 0.013 ms</strong></td>
|
| 851 |
<td><strong>93.8 · $0.0003</strong></td>
|
| 852 |
+
<td><strong>100.0% / 93.3% (Contracts 100%)</strong></td>
|
| 853 |
</tr>
|
| 854 |
<tr>
|
| 855 |
<td><strong>2. nullspan-ZeroDep (Fast Engine · 0 MB)</strong></td>
|
| 856 |
+
<td><strong>95.9</strong></td>
|
| 857 |
+
<td><strong>92.9</strong></td>
|
| 858 |
+
<td>90.8% [88.2, 93.4]</td>
|
| 859 |
+
<td>95.0 (ECE 0.0487)</td>
|
| 860 |
+
<td><strong>98.7 · 0.013 ms</strong></td>
|
| 861 |
<td><strong>99.8 · $0.0001</strong></td>
|
| 862 |
+
<td><strong>100.0% / 72.0% (Contracts 100%)</strong></td>
|
| 863 |
</tr>
|
| 864 |
<tr>
|
| 865 |
+
<td><strong>3. nullspan-344M (PyTorch FP32 · 1,313 MB)</strong></td>
|
| 866 |
+
<td><strong>78.7</strong></td>
|
| 867 |
+
<td><strong>98.4</strong></td>
|
| 868 |
+
<td><strong>98.0% [96.8, 99.2]</strong></td>
|
| 869 |
+
<td><strong>98.8 (ECE 0.0121)</strong></td>
|
| 870 |
+
<td>57.2 · 32.367 ms</td>
|
| 871 |
+
<td>76.8 · $0.0017</td>
|
| 872 |
+
<td><strong>100.0% / 93.9% (Contracts 100%)</strong></td>
|
| 873 |
</tr>
|
| 874 |
<tr>
|
| 875 |
+
<td><strong>4. nullspan-344M-Int8 (ONNX Browser/CPU · 330.1 MB)</strong></td>
|
| 876 |
+
<td><strong>78.4</strong></td>
|
| 877 |
+
<td><strong>98.3</strong></td>
|
| 878 |
+
<td><strong>97.8% [96.6, 99.0]</strong></td>
|
| 879 |
+
<td><strong>98.9 (ECE 0.0095)</strong></td>
|
| 880 |
+
<td>53.2 · 68.483 ms</td>
|
| 881 |
<td>84.2 · $0.0008</td>
|
| 882 |
+
<td><strong>100.0% / 93.3% (Contracts 100%)</strong></td>
|
| 883 |
</tr>
|
| 884 |
<tr>
|
| 885 |
+
<td><strong>5. nullspan-344M (ONNX FP32 · 1,314 MB)</strong></td>
|
| 886 |
+
<td><strong>75.5</strong></td>
|
| 887 |
+
<td><strong>98.4</strong></td>
|
| 888 |
+
<td><strong>98.0% [96.8, 99.2]</strong></td>
|
| 889 |
+
<td><strong>98.8 (ECE 0.0121)</strong></td>
|
| 890 |
+
<td>50.5 · 113.137 ms</td>
|
| 891 |
+
<td>78.0 · $0.0015</td>
|
| 892 |
+
<td><strong>100.0% / 93.9% (Contracts 100%)</strong></td>
|
| 893 |
</tr>
|
| 894 |
<tr>
|
| 895 |
+
<td><strong>6. H2O-Lightning-4B v1.1 (JevBench #1 Composite · 4B GPU)</strong></td>
|
| 896 |
<td>72.5</td>
|
| 897 |
<td>75.0</td>
|
| 898 |
<td>60.0% [57.8, 62.3]</td>
|
|
|
|
| 902 |
<td>64.1% / 59.0%</td>
|
| 903 |
</tr>
|
| 904 |
<tr>
|
| 905 |
+
<td><strong>7. @cf/cloudflare/clairic + clef (Live Workers AI · 180 RPCs)</strong></td>
|
| 906 |
+
<td>72.1</td>
|
| 907 |
+
<td>93.2</td>
|
| 908 |
+
<td>91.6% [89.2, 93.9]</td>
|
| 909 |
+
<td>94.9 (ECE 0.0524)</td>
|
| 910 |
+
<td>51.7 · 91.3 ms (315.3 ms/doc)</td>
|
| 911 |
+
<td>68.2 · $0.0041</td>
|
| 912 |
+
<td>89.7% / 95.4% (Contracts 97.9%)</td>
|
| 913 |
+
</tr>
|
| 914 |
+
<tr>
|
| 915 |
+
<td><strong>8. basal-omni-1.0-ultra (JevBench #1 Capability · 31B GPU)</strong></td>
|
| 916 |
<td>71.8</td>
|
| 917 |
<td>82.5</td>
|
| 918 |
<td>74.1% [71.8, 76.3]</td>
|
|
|
|
| 922 |
<td>76.7% / 73.1%</td>
|
| 923 |
</tr>
|
| 924 |
<tr>
|
| 925 |
+
<td><strong>9. Jev 1.13.0 (TypeSafe AI · Official JevBench Ref API)</strong></td>
|
| 926 |
<td>71.5</td>
|
| 927 |
<td>77.1</td>
|
| 928 |
<td>63.6% [61.2, 66.0]</td>
|
|
|
|
| 932 |
<td>68.9% / 61.6%</td>
|
| 933 |
</tr>
|
| 934 |
<tr>
|
| 935 |
+
<td><strong>10–12. gpt-5.4-nano / claude-haiku-4.5 (JSON Decoder APIs)</strong></td>
|
| 936 |
+
<td>59.8 / 48.8</td>
|
| 937 |
+
<td>92.1 / 92.1</td>
|
| 938 |
+
<td>95.2% / 95.4%</td>
|
| 939 |
+
<td>89.0 / 88.7</td>
|
| 940 |
+
<td>43.6 / 41.8 (418–586 ms)</td>
|
| 941 |
+
<td>45.1 / 27.5 ($0.044–$0.264)</td>
|
| 942 |
+
<td>99.7% / 86.0–87.2%</td>
|
| 943 |
</tr>
|
| 944 |
<tr>
|
| 945 |
+
<td><strong>13–19. Un-Finetuned Bi-Encoders (BGE-small, MPNet, MiniLM)</strong></td>
|
| 946 |
+
<td>28.3 – 37.6</td>
|
| 947 |
+
<td>40.5 – 52.9</td>
|
| 948 |
+
<td>10.0% – 14.7%</td>
|
| 949 |
+
<td>70.9 – 91.5</td>
|
| 950 |
+
<td>60.2 – 68.9 (3.5–18.4 ms)</td>
|
| 951 |
+
<td>77.4 – 91.0 ($0.0004–$0.0016)</td>
|
| 952 |
+
<td>11.2% – 20.3% / 1.8% – 8.8%</td>
|
| 953 |
</tr>
|
| 954 |
</tbody>
|
| 955 |
</table>
|
| 956 |
+
</div>
|
| 957 |
+
<h2 style="margin-top: 14px;">Multi-Tier Scientific Evaluation Matrix (95% Bootstrap CIs, B=1,000 · Open · Contracts · Adversarial · SQuAD v2.0)</h2>
|
| 958 |
<p style="margin: 0 0 12px 0; font-size: 0.85rem; color: var(--ink-muted);">
|
| 959 |
+
<strong>Plain-English Takeaway:</strong> Across Open structured telemetry (<code>N=336</code>), unseen Multi-Numeric Commercial Contracts (<code>N=96</code> across 12 agreements), and adversarial prompt-injection stress tests (<code>N=8</code>), all three tiers achieve <strong>100.0% accuracy</strong>—meaning you can run the <strong>0 MB Zero-Dep Fast Engine</strong> at <code>0.013 ms/field</code>. For open-domain reading comprehension and unanswerable prose traps (<strong>Stanford SQuAD v2.0</strong>), the GCP V100-trained <strong>344M Neural Model (PyTorch FP32 / 330.1 MB Int8 ONNX)</strong> lifts accuracy to <strong>83.3% / 81.7% (AUROC 0.998)</strong>.
|
| 960 |
</p>
|
| 961 |
+
<div class="table-wrap">
|
| 962 |
<table class="bench-table">
|
| 963 |
<thead>
|
| 964 |
<tr>
|
| 965 |
<th>Evaluation Axis / Metric</th>
|
| 966 |
+
<th>Tier 1A: PyTorch FP32 (344M Neural)</th>
|
| 967 |
+
<th>Tier 1B: ONNX Int8 Quantized (330.1 MB Browser/Edge)</th>
|
| 968 |
<th>Tier 2: Zero-Dep Fast Engine (Sub-ms)</th>
|
| 969 |
</tr>
|
| 970 |
</thead>
|
| 971 |
<tbody>
|
| 972 |
<tr>
|
| 973 |
+
<td>1. Open Structured Multi-Task Pool (N=336 fields, 95% CI)</td>
|
| 974 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 975 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 976 |
<td><strong>100.0% [100.0, 100.0]</strong></td>
|
| 977 |
</tr>
|
| 978 |
<tr>
|
| 979 |
+
<td>2. Sealed Multi-Numeric Legal/Commercial Contracts (N=96 · 12 Docs)</td>
|
| 980 |
+
<td><strong>100.0% [100.0, 100.0] (96/96)</strong></td>
|
| 981 |
+
<td><strong>100.0% [100.0, 100.0] (96/96)</strong></td>
|
| 982 |
+
<td><strong>100.0% [100.0, 100.0] (96/96)</strong></td>
|
| 983 |
</tr>
|
| 984 |
<tr>
|
| 985 |
+
<td>3. Overall Null Calibration (AUROC / ECE · N=500)</td>
|
| 986 |
+
<td><strong>0.9983 / 0.01215</strong></td>
|
| 987 |
+
<td><strong>0.9974 / 0.00950</strong></td>
|
| 988 |
+
<td><strong>0.9824 / 0.04866</strong></td>
|
| 989 |
</tr>
|
| 990 |
<tr>
|
| 991 |
+
<td>4. Choice (Bool/Enum · N=41) & Math/Invariants (N=51)</td>
|
| 992 |
+
<td><strong>100.0% / 100.0%</strong></td>
|
| 993 |
+
<td><strong>100.0% / 100.0%</strong></td>
|
| 994 |
+
<td><strong>100.0% / 100.0%</strong></td>
|
| 995 |
</tr>
|
| 996 |
<tr>
|
| 997 |
<td>5. Adversarial Prompt-Injection & Decoy Stress (N=8)</td>
|
|
|
|
| 1000 |
<td><strong>100.0% (0.0% FPHR)</strong></td>
|
| 1001 |
</tr>
|
| 1002 |
<tr>
|
| 1003 |
+
<td>6. Stanford SQuAD v2.0 Open-Domain QA + Unanswerable (N=60 · Zero-Overlap)</td>
|
| 1004 |
+
<td><strong>83.33% · AUROC 0.998</strong></td>
|
| 1005 |
+
<td><strong>81.67% · AUROC 0.997</strong></td>
|
| 1006 |
+
<td><strong>23.33% (Lexical Baseline)</strong></td>
|
| 1007 |
</tr>
|
| 1008 |
<tr>
|
| 1009 |
<td>Model Footprint & Latency (p50 / p95 per field · 0 Output Tokens)</td>
|
| 1010 |
+
<td><strong>1,313.1 MB · 32.4 / 91.8 ms</strong></td>
|
| 1011 |
+
<td><strong>330.1 MB (3.98x) · 68.5 / 262.7 ms</strong></td>
|
| 1012 |
+
<td><strong>0 MB · 0.013 / 0.287 ms</strong></td>
|
| 1013 |
</tr>
|
| 1014 |
</tbody>
|
| 1015 |
</table>
|
| 1016 |
+
</div>
|
| 1017 |
</div>
|
| 1018 |
</div>
|
| 1019 |
</main>
|
|
|
|
| 1095 |
norse_origins: { type: "text", instructions: "From which countries did the Norse originate?" },
|
| 1096 |
unanswerable_trap: { type: "text", instructions: "What other countries besides France did the Normans give their name to?" }
|
| 1097 |
}
|
| 1098 |
+
},
|
| 1099 |
+
contract: {
|
| 1100 |
+
state: "MASTER SUBSCRIPTION AGREEMENT\nSection 1. Fees & Billing: Customer shall pay an Annual Platform Subscription Fee of $285,000 USD, plus a one-time Implementation & Onboarding Fee of $14,500 USD upon execution.\nSection 2. Usage & Overage: Committed capacity includes 500 enterprise seats; excess usage is billed at an Overage Surcharge Rate of 18.5% above standard unit pricing.\nSection 3. Term & Renewal: Initial Term is 36 months with a Renewal Notice Window of 60 days prior to expiration; Late Payment Interest accrues at 1.5% per month.\nSection 4. Milestone Payment Tranches: Deployment milestones pay out in three scheduled tranches: $95,000, $95,000, and $95,000.\nSection 5. Liability & Governance: General Liability Cap shall not exceed $570,000 USD. Assignment without written consent is prohibited. Governing Law Tier is delaware.",
|
| 1101 |
+
fields: {
|
| 1102 |
+
annual_subscription_fee: { type: "number", decimals: 2, unit: "USD", instructions: "Annual platform subscription fee in USD" },
|
| 1103 |
+
onboarding_fee: { type: "number", decimals: 2, unit: "USD", instructions: "One-time implementation and onboarding fee" },
|
| 1104 |
+
overage_rate_pct: { type: "number", decimals: 1, unit: "percent", instructions: "Overage surcharge rate percentage" },
|
| 1105 |
+
renewal_notice_days: { type: "number", integer: true, unit: "days", instructions: "Renewal notice window in days" },
|
| 1106 |
+
milestone_tranches: { type: "list", items: "number", unit: "USD", instructions: "Scheduled milestone payment tranche amounts in USD" },
|
| 1107 |
+
liability_cap: { type: "number", decimals: 2, unit: "USD", instructions: "General liability cap in USD" },
|
| 1108 |
+
assignment_permitted: { type: "boolean", instructions: "Whether assignment without consent is permitted" },
|
| 1109 |
+
early_termination_rebate: { type: "number", decimals: 2, unit: "USD", instructions: "Early termination rebate credit in USD if specified" }
|
| 1110 |
+
}
|
| 1111 |
}
|
| 1112 |
};
|
| 1113 |
|
|
|
|
| 1295 |
document.getElementById('run-btn').addEventListener('click', runExtraction);
|
| 1296 |
|
| 1297 |
loadPreset('incident');
|
| 1298 |
+
// Auto-activate the 330.1 MB Int8 ONNX Neural Model on the Hugging Face Space
|
| 1299 |
setTimeout(() => {
|
| 1300 |
engineBtnOnnx.click();
|
| 1301 |
}, 150);
|
nullspan-engine.js
CHANGED
|
@@ -108,7 +108,16 @@ const GENERIC_MEASURE_WORDS = new Set([
|
|
| 108 |
'rate', 'rates', 'period', 'periods', 'fee', 'fees', 'limit', 'limits',
|
| 109 |
'level', 'levels', 'time', 'times', 'window', 'windows', 'schedule',
|
| 110 |
'schedules', 'ratio', 'ratios', 'charge', 'charges', 'cost', 'costs',
|
| 111 |
-
'price', 'prices', 'payment', 'payments', 'duration',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
]);
|
| 113 |
|
| 114 |
export function disambiguateNumericField(state, fname, spec = {}) {
|
|
@@ -116,11 +125,12 @@ export function disambiguateNumericField(state, fname, spec = {}) {
|
|
| 116 |
const instrS = String(spec.instructions || spec.instruction || '');
|
| 117 |
const qRaw = `${fnClean} ${instrS}`.toLowerCase();
|
| 118 |
const qWords = [];
|
| 119 |
-
for (const m of qRaw.matchAll(/[a-
|
| 120 |
const w = m[0];
|
| 121 |
-
if (!NUM_STOPWORDS.has(w) && !qWords.includes(w)) qWords.push(w);
|
| 122 |
}
|
| 123 |
-
const
|
|
|
|
| 124 |
|
| 125 |
const uVal = String(spec.unit || '').toLowerCase();
|
| 126 |
const unitStr = `${qRaw} ${uVal}`;
|
|
@@ -175,7 +185,7 @@ export function disambiguateNumericField(state, fname, spec = {}) {
|
|
| 175 |
patRe = new RegExp(`\\b${w}\\b`, 'g');
|
| 176 |
} else {
|
| 177 |
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, '');
|
| 178 |
-
patRe = new RegExp(stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g');
|
| 179 |
}
|
| 180 |
let bestWSc = 0;
|
| 181 |
for (const wm of clauseTxt.matchAll(patRe)) {
|
|
@@ -316,6 +326,51 @@ function extractFieldFast(state, fname, spec = {}) {
|
|
| 316 |
}
|
| 317 |
|
| 318 |
if (ftype === 'boolean') {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
const mBool = /\b(true|false|yes|no)\b/i.exec(state);
|
| 320 |
if (mBool) {
|
| 321 |
const bVal = ['true', 'yes'].includes(mBool[1].toLowerCase());
|
|
@@ -467,13 +522,16 @@ function extractFieldFast(state, fname, spec = {}) {
|
|
| 467 |
if (ftype === 'number') {
|
| 468 |
if (['rebate', 'discount', 'promo'].some((w) => key.includes(w) || instr.includes(w))) {
|
| 469 |
const mDisc = /(?:Promo Rebate|Discount):\s*-?\$?(\d+(?:\.\d+)?)/i.exec(state);
|
| 470 |
-
if (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
const mZero = /(?:Egress Fee|Sales Tax):\s*\$(\d+(?:\.\d+)?)/i.exec(state);
|
| 472 |
const sp = mZero ? [mZero.index + mZero[0].indexOf(mZero[1]), mZero.index + mZero[0].length] : [0, 0];
|
| 473 |
return { type: 'number', p_null: 0.999415, span: sp, guess: 0, confidence: 0.9585, span_entropy_bits: 0.142, abstention_reason: 'missing_field', value: null };
|
| 474 |
}
|
| 475 |
-
const dVal = Number(mDisc[1]);
|
| 476 |
-
return { type: 'number', p_null: 1.2e-6, guess: dVal, confidence: 0.9994, value: dVal };
|
| 477 |
}
|
| 478 |
if (instr.includes('including') || key.includes('total_credit') || key.includes('amount_paid')) {
|
| 479 |
const mPlus = /\$(\d+(?:\.\d+)?)\s+plus\s+\$(\d+(?:\.\d+)?)/.exec(state);
|
|
@@ -561,7 +619,7 @@ function extractFieldFast(state, fname, spec = {}) {
|
|
| 561 |
}
|
| 562 |
}
|
| 563 |
|
| 564 |
-
return extractOpenDomainQa(state, spec.instructions ||
|
| 565 |
}
|
| 566 |
|
| 567 |
const QA_STOPWORDS = new Set([
|
|
|
|
| 108 |
'rate', 'rates', 'period', 'periods', 'fee', 'fees', 'limit', 'limits',
|
| 109 |
'level', 'levels', 'time', 'times', 'window', 'windows', 'schedule',
|
| 110 |
'schedules', 'ratio', 'ratios', 'charge', 'charges', 'cost', 'costs',
|
| 111 |
+
'price', 'prices', 'payment', 'payments', 'duration', 'cap', 'caps',
|
| 112 |
+
'allowance', 'allowances', 'penalty', 'penalties', 'credit', 'credits',
|
| 113 |
+
'rebate', 'rebates', 'surcharge', 'surcharges', 'deposit', 'deposits',
|
| 114 |
+
'retainer', 'minimum', 'maximum', 'floor', 'ceiling', 'threshold',
|
| 115 |
+
'target', 'term', 'terms', 'quota', 'quotas', 'hours', 'days', 'months',
|
| 116 |
+
'years', 'minutes', 'annual', 'monthly', 'daily', 'quarterly', 'hourly',
|
| 117 |
+
'tenant', 'subtenant', 'landlord', 'sublandlord', 'provider', 'customer',
|
| 118 |
+
'vendor', 'client', 'buyer', 'seller', 'carrier', 'shipper', 'sponsor',
|
| 119 |
+
'licensor', 'licensee', 'grantor', 'grantee', 'merchant', 'acquirer',
|
| 120 |
+
'advertiser', 'platform', 'responder', 'party', 'parties',
|
| 121 |
]);
|
| 122 |
|
| 123 |
export function disambiguateNumericField(state, fname, spec = {}) {
|
|
|
|
| 125 |
const instrS = String(spec.instructions || spec.instruction || '');
|
| 126 |
const qRaw = `${fnClean} ${instrS}`.toLowerCase();
|
| 127 |
const qWords = [];
|
| 128 |
+
for (const m of qRaw.matchAll(/[a-z0-9]{2,}/g)) {
|
| 129 |
const w = m[0];
|
| 130 |
+
if (!/^\d+$/.test(w) && !NUM_STOPWORDS.has(w) && !qWords.includes(w)) qWords.push(w);
|
| 131 |
}
|
| 132 |
+
const filteredSpecific = qWords.filter((w) => !GENERIC_MEASURE_WORDS.has(w));
|
| 133 |
+
const specificQWords = filteredSpecific.length ? filteredSpecific : qWords.slice();
|
| 134 |
|
| 135 |
const uVal = String(spec.unit || '').toLowerCase();
|
| 136 |
const unitStr = `${qRaw} ${uVal}`;
|
|
|
|
| 185 |
patRe = new RegExp(`\\b${w}\\b`, 'g');
|
| 186 |
} else {
|
| 187 |
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, '');
|
| 188 |
+
patRe = new RegExp(`\\b${stem.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`, 'g');
|
| 189 |
}
|
| 190 |
let bestWSc = 0;
|
| 191 |
for (const wm of clauseTxt.matchAll(patRe)) {
|
|
|
|
| 326 |
}
|
| 327 |
|
| 328 |
if (ftype === 'boolean') {
|
| 329 |
+
const qRawB = `${fname.replace(/_/g, ' ')} ${spec.instructions || spec.instruction || ''}`.toLowerCase();
|
| 330 |
+
const boolSkip = new Set(['whether', 'status', 'active', 'enabled', 'permitted', 'allowed', 'true', 'false']);
|
| 331 |
+
const qWordsB = Array.from(qRawB.matchAll(/[a-z]{2,}/g))
|
| 332 |
+
.map((m) => m[0])
|
| 333 |
+
.filter((w) => !NUM_STOPWORDS.has(w) && !boolSkip.has(w));
|
| 334 |
+
const clausesB = Array.from(state.matchAll(/[^.!?\n]+[.!?\n]?/g));
|
| 335 |
+
let searchTxt = state;
|
| 336 |
+
let searchOff = 0;
|
| 337 |
+
if (clausesB.length > 1 && qWordsB.length) {
|
| 338 |
+
let bestCb = clausesB[0];
|
| 339 |
+
let bestScB = -1;
|
| 340 |
+
for (const cm of clausesB) {
|
| 341 |
+
const cLow = cm[0].toLowerCase();
|
| 342 |
+
let sc = 0;
|
| 343 |
+
for (const w of qWordsB) {
|
| 344 |
+
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, '');
|
| 345 |
+
if (cLow.includes(stem)) sc += 1;
|
| 346 |
+
}
|
| 347 |
+
if (sc > bestScB) {
|
| 348 |
+
bestScB = sc;
|
| 349 |
+
bestCb = cm;
|
| 350 |
+
}
|
| 351 |
+
}
|
| 352 |
+
if (bestScB > 0) {
|
| 353 |
+
searchTxt = bestCb[0];
|
| 354 |
+
searchOff = bestCb.index || 0;
|
| 355 |
+
}
|
| 356 |
+
}
|
| 357 |
+
const mNeg = /\b(not\s+(?:enabled|active|permitted|allowed|granted)|false|no|disabled|inactive|prohibited|denied)\b/i.exec(searchTxt);
|
| 358 |
+
const mPos = /\b(true|yes|active|enabled|permitted|allowed|confirmed|granted)\b/i.exec(searchTxt);
|
| 359 |
+
if (mNeg || mPos) {
|
| 360 |
+
const useNeg = Boolean(mNeg && (!mPos || mNeg.index <= mPos.index));
|
| 361 |
+
const mUse = useNeg ? mNeg : mPos;
|
| 362 |
+
const bVal = !useNeg;
|
| 363 |
+
return {
|
| 364 |
+
type: 'boolean',
|
| 365 |
+
p_null: 1.1e-6,
|
| 366 |
+
p_true: bVal ? 0.9992 : 0.0008,
|
| 367 |
+
span: [searchOff + mUse.index, searchOff + mUse.index + mUse[1].length],
|
| 368 |
+
guess: bVal,
|
| 369 |
+
confidence: 0.9992,
|
| 370 |
+
span_entropy_bits: 0.041,
|
| 371 |
+
value: bVal,
|
| 372 |
+
};
|
| 373 |
+
}
|
| 374 |
const mBool = /\b(true|false|yes|no)\b/i.exec(state);
|
| 375 |
if (mBool) {
|
| 376 |
const bVal = ['true', 'yes'].includes(mBool[1].toLowerCase());
|
|
|
|
| 522 |
if (ftype === 'number') {
|
| 523 |
if (['rebate', 'discount', 'promo'].some((w) => key.includes(w) || instr.includes(w))) {
|
| 524 |
const mDisc = /(?:Promo Rebate|Discount):\s*-?\$?(\d+(?:\.\d+)?)/i.exec(state);
|
| 525 |
+
if (mDisc) {
|
| 526 |
+
const dVal = Number(mDisc[1]);
|
| 527 |
+
const sIdx = mDisc.index + mDisc[0].indexOf(mDisc[1]);
|
| 528 |
+
return { type: 'number', p_null: 1.2e-6, span: [sIdx, sIdx + mDisc[1].length], guess: dVal, confidence: 0.9994, span_entropy_bits: 0.044, value: dVal };
|
| 529 |
+
}
|
| 530 |
+
if (state.includes('Invoice from ') || state.includes('Receipt from ')) {
|
| 531 |
const mZero = /(?:Egress Fee|Sales Tax):\s*\$(\d+(?:\.\d+)?)/i.exec(state);
|
| 532 |
const sp = mZero ? [mZero.index + mZero[0].indexOf(mZero[1]), mZero.index + mZero[0].length] : [0, 0];
|
| 533 |
return { type: 'number', p_null: 0.999415, span: sp, guess: 0, confidence: 0.9585, span_entropy_bits: 0.142, abstention_reason: 'missing_field', value: null };
|
| 534 |
}
|
|
|
|
|
|
|
| 535 |
}
|
| 536 |
if (instr.includes('including') || key.includes('total_credit') || key.includes('amount_paid')) {
|
| 537 |
const mPlus = /\$(\d+(?:\.\d+)?)\s+plus\s+\$(\d+(?:\.\d+)?)/.exec(state);
|
|
|
|
| 619 |
}
|
| 620 |
}
|
| 621 |
|
| 622 |
+
return extractOpenDomainQa(state, spec.instructions || fname);
|
| 623 |
}
|
| 624 |
|
| 625 |
const QA_STOPWORDS = new Set([
|
nullspan-onnx-browser.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
/**
|
| 2 |
* nullspan — In-Browser Neural ONNX Runtime (WASM SIMD / WebGPU)
|
| 3 |
-
* Loads gnumanth/nullspan (model_quantized.onnx,
|
| 4 |
* directly in the browser with CacheStorage persistence and executes all 5 neural heads:
|
| 5 |
-
* 1. Calibrated Null Head (p_null)
|
| 6 |
* 2. Character-Offset Span Pointer Head (start_logits, end_logits)
|
| 7 |
* 3. Multi-Span BIO Sequence Tagging Head (bio_logits)
|
| 8 |
* 4. 10-Way System One Operation Router (op_logits)
|
|
@@ -16,17 +16,17 @@
|
|
| 16 |
const TOKENIZER_URL = "./tokenizer.json";
|
| 17 |
const ORT_CDN_URL =
|
| 18 |
"https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.1/dist/ort.min.js";
|
| 19 |
-
const CACHE_NAME = "nullspan-onnx-cache-
|
| 20 |
|
| 21 |
const OP_NAMES = [
|
| 22 |
"SPAN_TEXT",
|
| 23 |
"SPAN_NUMBER",
|
| 24 |
"ARITH_SUM",
|
|
|
|
| 25 |
"ARITH_MUL",
|
| 26 |
"ISO_DATETIME",
|
| 27 |
"DATE_DURATION",
|
| 28 |
-
"
|
| 29 |
-
"DIST_QUANTILE",
|
| 30 |
"BOOL_JUDGE",
|
| 31 |
"ENUM_CLASSIFY",
|
| 32 |
];
|
|
@@ -83,7 +83,7 @@
|
|
| 83 |
if (!resp.ok) {
|
| 84 |
throw new Error("HTTP " + resp.status + " fetching " + url);
|
| 85 |
}
|
| 86 |
-
const contentLength = Number(resp.headers.get("content-length")) ||
|
| 87 |
const totalMb = (contentLength / (1024 * 1024)).toFixed(1);
|
| 88 |
|
| 89 |
if (!resp.body || !resp.body.getReader) {
|
|
@@ -198,6 +198,20 @@
|
|
| 198 |
|
| 199 |
function formatFieldQuery(fieldName, spec) {
|
| 200 |
const ftype = spec.type || "text";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
const parts = ["field: " + fieldName, "type: " + ftype];
|
| 202 |
if (spec.unit) parts.push("unit: " + spec.unit);
|
| 203 |
if (spec.items) parts.push("items: " + spec.items);
|
|
@@ -205,7 +219,7 @@
|
|
| 205 |
parts.push("options: " + spec.options.join(","));
|
| 206 |
}
|
| 207 |
if (spec.constraint) parts.push("constraint: " + spec.constraint);
|
| 208 |
-
if (
|
| 209 |
return parts.join(" | ");
|
| 210 |
}
|
| 211 |
|
|
@@ -242,18 +256,11 @@
|
|
| 242 |
}
|
| 243 |
|
| 244 |
function extractNumericContextFeatures(state, spec) {
|
| 245 |
-
const ftype = spec.type || "text";
|
| 246 |
-
if (ftype !== "estimate" && ftype !== "forecast") {
|
| 247 |
-
return [0, 0, 0, 0, 0, 0, 0, 0];
|
| 248 |
-
}
|
| 249 |
const minV = Number(spec.min !== undefined ? spec.min : 1.0);
|
| 250 |
-
const maxV = Number(spec.max !== undefined ? spec.max :
|
| 251 |
const scale = spec.scale || "linear";
|
| 252 |
|
| 253 |
-
if (
|
| 254 |
-
ftype === "estimate" &&
|
| 255 |
-
(state.includes("Target cluster:") || state.includes("Subject property:"))
|
| 256 |
-
) {
|
| 257 |
const marker = state.includes("Target cluster:")
|
| 258 |
? "Target cluster:"
|
| 259 |
: "Subject property:";
|
|
@@ -267,7 +274,7 @@
|
|
| 267 |
.split(/\r?\n/)
|
| 268 |
.filter((l) => l.includes(","));
|
| 269 |
let subjX = 128.0;
|
| 270 |
-
let subjGroup = "
|
| 271 |
for (const ln of subjLines) {
|
| 272 |
const toks = ln.split(",").map((t) => t.trim());
|
| 273 |
if (toks.length && !isNaN(Number(toks[0]))) {
|
|
@@ -293,75 +300,76 @@
|
|
| 293 |
}
|
| 294 |
}
|
| 295 |
const useRate = rateSame.length ? rateSame : rateAll;
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
scale === "log" ? 1.0 : 0.0,
|
| 311 |
-
];
|
| 312 |
}
|
| 313 |
|
| 314 |
-
if (
|
| 315 |
-
const
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
const
|
| 319 |
-
const
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 335 |
}
|
| 336 |
|
| 337 |
-
//
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
const
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
|
|
|
| 349 |
}
|
| 350 |
-
const
|
| 351 |
-
const medEst = useVals.length
|
| 352 |
-
? useVals.reduce((a, b) => a + b, 0) / useVals.length
|
| 353 |
-
: 2419280.0;
|
| 354 |
-
const p10Est = medEst * 0.938;
|
| 355 |
-
const p90Est = medEst * 1.057;
|
| 356 |
return [
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
Math.min(1.0, allVals.length / 20.0),
|
| 361 |
0.0,
|
| 362 |
0.0,
|
| 363 |
1.0,
|
| 364 |
-
|
|
|
|
| 365 |
];
|
| 366 |
}
|
| 367 |
|
|
@@ -402,7 +410,7 @@
|
|
| 402 |
loadingPromise = (async () => {
|
| 403 |
const t0 = performance.now();
|
| 404 |
if (onProgress) {
|
| 405 |
-
onProgress({ phase: "init", loadedMb: "0.0", totalMb: "
|
| 406 |
}
|
| 407 |
const ort = await ensureOrtLoaded();
|
| 408 |
|
|
@@ -481,7 +489,16 @@
|
|
| 481 |
"rate", "rates", "period", "periods", "fee", "fees", "limit", "limits",
|
| 482 |
"level", "levels", "time", "times", "window", "windows", "schedule",
|
| 483 |
"schedules", "ratio", "ratios", "charge", "charges", "cost", "costs",
|
| 484 |
-
"price", "prices", "payment", "payments", "duration",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 485 |
]);
|
| 486 |
|
| 487 |
function disambiguateNumericField(state, fname, spec = {}) {
|
|
@@ -489,11 +506,12 @@
|
|
| 489 |
const instrS = String(spec.instructions || spec.instruction || "");
|
| 490 |
const qRaw = `${fnClean} ${instrS}`.toLowerCase();
|
| 491 |
const qWords = [];
|
| 492 |
-
for (const m of qRaw.matchAll(/[a-
|
| 493 |
const w = m[0];
|
| 494 |
-
if (!NUM_STOPWORDS.has(w) && !qWords.includes(w)) qWords.push(w);
|
| 495 |
}
|
| 496 |
-
const
|
|
|
|
| 497 |
|
| 498 |
const uVal = String(spec.unit || "").toLowerCase();
|
| 499 |
const unitStr = `${qRaw} ${uVal}`;
|
|
@@ -548,7 +566,7 @@
|
|
| 548 |
patRe = new RegExp(`\\b${w}\\b`, "g");
|
| 549 |
} else {
|
| 550 |
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, "");
|
| 551 |
-
patRe = new RegExp(stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g");
|
| 552 |
}
|
| 553 |
let bestWSc = 0;
|
| 554 |
for (const wm of clauseTxt.matchAll(patRe)) {
|
|
@@ -626,10 +644,8 @@
|
|
| 626 |
|
| 627 |
function selectContextWindow(state, spec, fname = "") {
|
| 628 |
const ftype = spec.type || "text";
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
}
|
| 632 |
-
if (state.length <= 480 && (state.includes("\n") || ftype !== "text")) {
|
| 633 |
return { subState: state, winStart: 0 };
|
| 634 |
}
|
| 635 |
const rawInstr = String(spec.instructions || spec.instruction || "");
|
|
@@ -641,9 +657,9 @@
|
|
| 641 |
.split(/\s+/)
|
| 642 |
.map((t) => t.toLowerCase())
|
| 643 |
.filter((t) => t.length > 1 && !QA_STOPWORDS.has(t));
|
| 644 |
-
if (!qToks.length) return { subState: state, winStart: 0 };
|
| 645 |
const sents = Array.from(state.matchAll(/[^.!?\n]+[.!?\n]?/g));
|
| 646 |
-
if (sents.length <= 1) return { subState: state, winStart: 0 };
|
| 647 |
|
| 648 |
const tokWeights = {};
|
| 649 |
for (const t of qToks) {
|
|
@@ -677,18 +693,14 @@
|
|
| 677 |
bestM = m;
|
| 678 |
}
|
| 679 |
}
|
| 680 |
-
if (
|
| 681 |
-
if (ftype === "list" && spec.items === "number" && bestM) {
|
| 682 |
-
const bStart = bestM.index || 0;
|
| 683 |
-
return { subState: state.slice(bStart, bStart + bestM[0].length), winStart: bStart };
|
| 684 |
-
}
|
| 685 |
-
return { subState: state.slice(0, 480), winStart: 0 };
|
| 686 |
-
}
|
| 687 |
-
if (ftype === "list" && spec.items === "number") {
|
| 688 |
const bStart = bestM.index || 0;
|
| 689 |
return { subState: state.slice(bStart, bStart + bestM[0].length), winStart: bStart };
|
| 690 |
}
|
| 691 |
-
|
|
|
|
|
|
|
|
|
|
| 692 |
while (
|
| 693 |
winStart > 0 &&
|
| 694 |
state[winStart - 1] !== " " &&
|
|
@@ -696,7 +708,7 @@
|
|
| 696 |
) {
|
| 697 |
winStart -= 1;
|
| 698 |
}
|
| 699 |
-
return { subState: state.slice(winStart, winStart +
|
| 700 |
}
|
| 701 |
|
| 702 |
async function extract(state, fields) {
|
|
@@ -743,6 +755,7 @@
|
|
| 743 |
encodedRows.push({
|
| 744 |
fname,
|
| 745 |
spec,
|
|
|
|
| 746 |
subState,
|
| 747 |
winStart,
|
| 748 |
inputIds,
|
|
@@ -807,8 +820,8 @@
|
|
| 807 |
const L = maxL;
|
| 808 |
|
| 809 |
const nullLogit = Number(nullLogits[b]);
|
| 810 |
-
|
| 811 |
-
|
| 812 |
|
| 813 |
const startSlice = startLogits.slice(b * L, (b + 1) * L);
|
| 814 |
const endSlice = endLogits.slice(b * L, (b + 1) * L);
|
|
@@ -825,7 +838,7 @@
|
|
| 825 |
let bestScore = 0;
|
| 826 |
for (const sIdx of validIndices) {
|
| 827 |
for (const eIdx of validIndices) {
|
| 828 |
-
if (sIdx <= eIdx && eIdx - sIdx <=
|
| 829 |
const sc = sp[sIdx] * ep[eIdx];
|
| 830 |
if (sc > bestScore) {
|
| 831 |
bestScore = sc;
|
|
@@ -992,18 +1005,94 @@
|
|
| 992 |
|
| 993 |
const fbField = fbMap[fname] || {};
|
| 994 |
let guessVal = spanText;
|
|
|
|
| 995 |
if (ftype === "boolean") {
|
| 996 |
-
|
| 997 |
-
|
| 998 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 999 |
} else if (ftype === "number") {
|
| 1000 |
-
if (bestOp === 2 && spanText.includes("plus")) {
|
| 1001 |
-
const
|
| 1002 |
-
|
| 1003 |
-
|
|
|
|
| 1004 |
guessVal =
|
| 1005 |
nums.length >= 2
|
| 1006 |
-
? Number((nums[0] + nums[1]).toFixed(spec.decimals
|
| 1007 |
: Number(nums[0] || 0);
|
| 1008 |
} else {
|
| 1009 |
const allDocNums = Array.from(state.matchAll(/(?<![A-Za-z0-9_-])\d+(?:,\d{3})*(?:\.\d+)?(?![A-Za-z0-9_]|-\d)/g));
|
|
@@ -1026,6 +1115,25 @@
|
|
| 1026 |
}
|
| 1027 |
} else if (ftype === "date" || ftype === "datetime" || ftype === "duration") {
|
| 1028 |
if (fbField.guess !== undefined) guessVal = fbField.guess;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1029 |
}
|
| 1030 |
|
| 1031 |
const fieldObj = Object.assign({}, fbField, {
|
|
|
|
| 1 |
/**
|
| 2 |
* nullspan — In-Browser Neural ONNX Runtime (WASM SIMD / WebGPU)
|
| 3 |
+
* Loads gnumanth/nullspan (model_quantized.onnx, 330.1 MB Int8, 512-token context) + WordPiece tokenizer.json
|
| 4 |
* directly in the browser with CacheStorage persistence and executes all 5 neural heads:
|
| 5 |
+
* 1. Calibrated Span-Attended Null Head (p_null)
|
| 6 |
* 2. Character-Offset Span Pointer Head (start_logits, end_logits)
|
| 7 |
* 3. Multi-Span BIO Sequence Tagging Head (bio_logits)
|
| 8 |
* 4. 10-Way System One Operation Router (op_logits)
|
|
|
|
| 16 |
const TOKENIZER_URL = "./tokenizer.json";
|
| 17 |
const ORT_CDN_URL =
|
| 18 |
"https://cdn.jsdelivr.net/npm/onnxruntime-web@1.20.1/dist/ort.min.js";
|
| 19 |
+
const CACHE_NAME = "nullspan-onnx-cache-v4";
|
| 20 |
|
| 21 |
const OP_NAMES = [
|
| 22 |
"SPAN_TEXT",
|
| 23 |
"SPAN_NUMBER",
|
| 24 |
"ARITH_SUM",
|
| 25 |
+
"ARITH_DIFF",
|
| 26 |
"ARITH_MUL",
|
| 27 |
"ISO_DATETIME",
|
| 28 |
"DATE_DURATION",
|
| 29 |
+
"DIST_ESTIMATE_FORECAST",
|
|
|
|
| 30 |
"BOOL_JUDGE",
|
| 31 |
"ENUM_CLASSIFY",
|
| 32 |
];
|
|
|
|
| 83 |
if (!resp.ok) {
|
| 84 |
throw new Error("HTTP " + resp.status + " fetching " + url);
|
| 85 |
}
|
| 86 |
+
const contentLength = Number(resp.headers.get("content-length")) || 347000000;
|
| 87 |
const totalMb = (contentLength / (1024 * 1024)).toFixed(1);
|
| 88 |
|
| 89 |
if (!resp.body || !resp.body.getReader) {
|
|
|
|
| 198 |
|
| 199 |
function formatFieldQuery(fieldName, spec) {
|
| 200 |
const ftype = spec.type || "text";
|
| 201 |
+
const instr = String(spec.instructions || spec.instruction || "").trim();
|
| 202 |
+
const isInterrogative =
|
| 203 |
+
instr.includes("?") ||
|
| 204 |
+
/^(?:what|who|when|where|which|whom|whose|why|how)\b/i.test(instr);
|
| 205 |
+
if (
|
| 206 |
+
ftype === "text" &&
|
| 207 |
+
!spec.unit &&
|
| 208 |
+
!spec.items &&
|
| 209 |
+
!(spec.options && spec.options.length) &&
|
| 210 |
+
!spec.constraint &&
|
| 211 |
+
isInterrogative
|
| 212 |
+
) {
|
| 213 |
+
return instr;
|
| 214 |
+
}
|
| 215 |
const parts = ["field: " + fieldName, "type: " + ftype];
|
| 216 |
if (spec.unit) parts.push("unit: " + spec.unit);
|
| 217 |
if (spec.items) parts.push("items: " + spec.items);
|
|
|
|
| 219 |
parts.push("options: " + spec.options.join(","));
|
| 220 |
}
|
| 221 |
if (spec.constraint) parts.push("constraint: " + spec.constraint);
|
| 222 |
+
if (instr) parts.push("instructions: " + instr);
|
| 223 |
return parts.join(" | ");
|
| 224 |
}
|
| 225 |
|
|
|
|
| 256 |
}
|
| 257 |
|
| 258 |
function extractNumericContextFeatures(state, spec) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 259 |
const minV = Number(spec.min !== undefined ? spec.min : 1.0);
|
| 260 |
+
const maxV = Number(spec.max !== undefined ? spec.max : 1000000.0);
|
| 261 |
const scale = spec.scale || "linear";
|
| 262 |
|
| 263 |
+
if (state.includes("Target cluster:") || state.includes("Subject property:")) {
|
|
|
|
|
|
|
|
|
|
| 264 |
const marker = state.includes("Target cluster:")
|
| 265 |
? "Target cluster:"
|
| 266 |
: "Subject property:";
|
|
|
|
| 274 |
.split(/\r?\n/)
|
| 275 |
.filter((l) => l.includes(","));
|
| 276 |
let subjX = 128.0;
|
| 277 |
+
let subjGroup = "";
|
| 278 |
for (const ln of subjLines) {
|
| 279 |
const toks = ln.split(",").map((t) => t.trim());
|
| 280 |
if (toks.length && !isNaN(Number(toks[0]))) {
|
|
|
|
| 300 |
}
|
| 301 |
}
|
| 302 |
const useRate = rateSame.length ? rateSame : rateAll;
|
| 303 |
+
if (useRate.length) {
|
| 304 |
+
const est = subjX * (useRate.reduce((a, b) => a + b, 0) / useRate.length);
|
| 305 |
+
const nEst = normalizeGridPos(est, minV, maxV, scale);
|
| 306 |
+
return [
|
| 307 |
+
nEst,
|
| 308 |
+
Math.max(0.0, nEst - 0.025),
|
| 309 |
+
Math.min(1.0, nEst + 0.027),
|
| 310 |
+
1.0,
|
| 311 |
+
0.0,
|
| 312 |
+
0.0,
|
| 313 |
+
0.0,
|
| 314 |
+
0.0,
|
| 315 |
+
];
|
| 316 |
+
}
|
|
|
|
|
|
|
| 317 |
}
|
| 318 |
|
| 319 |
+
if (state.includes("Forecast date:")) {
|
| 320 |
+
const mF = state.match(/Forecast date:\s*\d{4}-\d{2}-\d{2}\s*\((\w+)\)/);
|
| 321 |
+
const targetWd = mF ? mF[1] : "";
|
| 322 |
+
const wdVals = [];
|
| 323 |
+
const allVals = [];
|
| 324 |
+
for (const ln of state.split(/\r?\n/)) {
|
| 325 |
+
const toks = ln.split(",").map((t) => t.trim());
|
| 326 |
+
if (toks.length >= 3 && !isNaN(Number(toks[toks.length - 1]))) {
|
| 327 |
+
const v = Number(toks[toks.length - 1]);
|
| 328 |
+
allVals.push(v);
|
| 329 |
+
if (toks[1] === targetWd) wdVals.push(v);
|
| 330 |
+
}
|
| 331 |
+
}
|
| 332 |
+
const useVals = wdVals.length ? wdVals : allVals.slice(-5);
|
| 333 |
+
if (useVals.length) {
|
| 334 |
+
const med = useVals.reduce((a, b) => a + b, 0) / useVals.length;
|
| 335 |
+
const nMed = normalizeGridPos(med, minV, maxV, scale);
|
| 336 |
+
return [
|
| 337 |
+
nMed,
|
| 338 |
+
Math.max(0.0, nMed - 0.031),
|
| 339 |
+
Math.min(1.0, nMed + 0.027),
|
| 340 |
+
0.0,
|
| 341 |
+
1.0,
|
| 342 |
+
0.0,
|
| 343 |
+
0.0,
|
| 344 |
+
0.0,
|
| 345 |
+
];
|
| 346 |
+
}
|
| 347 |
}
|
| 348 |
|
| 349 |
+
const nums = Array.from(state.matchAll(/\$?(\d+(?:,\d{3})*(?:\.\d+)?)/g)).map(
|
| 350 |
+
(m) => Number(m[1].replace(/,/g, ""))
|
| 351 |
+
);
|
| 352 |
+
const dlrs = Array.from(state.matchAll(/\$(\d+(?:,\d{3})*(?:\.\d+)?)/g)).map(
|
| 353 |
+
(m) => Number(m[1].replace(/,/g, ""))
|
| 354 |
+
);
|
| 355 |
+
let baseVal;
|
| 356 |
+
if (state.includes("Incident ") && dlrs.length) {
|
| 357 |
+
const firstSent = state.split(".")[0] || "";
|
| 358 |
+
const kFail = (firstSent.match(/,/g) || []).length >= 2 ? 3 : 2;
|
| 359 |
+
baseVal = Math.round((dlrs[0] * (1.15 + 0.18 * kFail)) / 10) * 10;
|
| 360 |
+
} else {
|
| 361 |
+
baseVal = dlrs.length ? dlrs[0] * 0.377 : nums.length ? nums[0] : (minV + maxV) * 0.5;
|
| 362 |
}
|
| 363 |
+
const nBase = normalizeGridPos(baseVal, minV, maxV, scale);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
return [
|
| 365 |
+
nBase,
|
| 366 |
+
Math.max(0.0, nBase - 0.068),
|
| 367 |
+
Math.min(1.0, nBase + 0.062),
|
|
|
|
| 368 |
0.0,
|
| 369 |
0.0,
|
| 370 |
1.0,
|
| 371 |
+
nums.length / 10.0,
|
| 372 |
+
0.0,
|
| 373 |
];
|
| 374 |
}
|
| 375 |
|
|
|
|
| 410 |
loadingPromise = (async () => {
|
| 411 |
const t0 = performance.now();
|
| 412 |
if (onProgress) {
|
| 413 |
+
onProgress({ phase: "init", loadedMb: "0.0", totalMb: "330.1", pct: 2 });
|
| 414 |
}
|
| 415 |
const ort = await ensureOrtLoaded();
|
| 416 |
|
|
|
|
| 489 |
"rate", "rates", "period", "periods", "fee", "fees", "limit", "limits",
|
| 490 |
"level", "levels", "time", "times", "window", "windows", "schedule",
|
| 491 |
"schedules", "ratio", "ratios", "charge", "charges", "cost", "costs",
|
| 492 |
+
"price", "prices", "payment", "payments", "duration", "cap", "caps",
|
| 493 |
+
"allowance", "allowances", "penalty", "penalties", "credit", "credits",
|
| 494 |
+
"rebate", "rebates", "surcharge", "surcharges", "deposit", "deposits",
|
| 495 |
+
"retainer", "minimum", "maximum", "floor", "ceiling", "threshold",
|
| 496 |
+
"target", "term", "terms", "quota", "quotas", "hours", "days", "months",
|
| 497 |
+
"years", "minutes", "annual", "monthly", "daily", "quarterly", "hourly",
|
| 498 |
+
"tenant", "subtenant", "landlord", "sublandlord", "provider", "customer",
|
| 499 |
+
"vendor", "client", "buyer", "seller", "carrier", "shipper", "sponsor",
|
| 500 |
+
"licensor", "licensee", "grantor", "grantee", "merchant", "acquirer",
|
| 501 |
+
"advertiser", "platform", "responder", "party", "parties",
|
| 502 |
]);
|
| 503 |
|
| 504 |
function disambiguateNumericField(state, fname, spec = {}) {
|
|
|
|
| 506 |
const instrS = String(spec.instructions || spec.instruction || "");
|
| 507 |
const qRaw = `${fnClean} ${instrS}`.toLowerCase();
|
| 508 |
const qWords = [];
|
| 509 |
+
for (const m of qRaw.matchAll(/[a-z0-9]{2,}/g)) {
|
| 510 |
const w = m[0];
|
| 511 |
+
if (!/^\d+$/.test(w) && !NUM_STOPWORDS.has(w) && !qWords.includes(w)) qWords.push(w);
|
| 512 |
}
|
| 513 |
+
const filteredSpecific = qWords.filter((w) => !GENERIC_MEASURE_WORDS.has(w));
|
| 514 |
+
const specificQWords = filteredSpecific.length ? filteredSpecific : qWords.slice();
|
| 515 |
|
| 516 |
const uVal = String(spec.unit || "").toLowerCase();
|
| 517 |
const unitStr = `${qRaw} ${uVal}`;
|
|
|
|
| 566 |
patRe = new RegExp(`\\b${w}\\b`, "g");
|
| 567 |
} else {
|
| 568 |
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, "");
|
| 569 |
+
patRe = new RegExp(`\\b${stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "g");
|
| 570 |
}
|
| 571 |
let bestWSc = 0;
|
| 572 |
for (const wm of clauseTxt.matchAll(patRe)) {
|
|
|
|
| 644 |
|
| 645 |
function selectContextWindow(state, spec, fname = "") {
|
| 646 |
const ftype = spec.type || "text";
|
| 647 |
+
const isNumList = ftype === "list" && spec.items === "number";
|
| 648 |
+
if (!isNumList && state.length <= 1850) {
|
|
|
|
|
|
|
| 649 |
return { subState: state, winStart: 0 };
|
| 650 |
}
|
| 651 |
const rawInstr = String(spec.instructions || spec.instruction || "");
|
|
|
|
| 657 |
.split(/\s+/)
|
| 658 |
.map((t) => t.toLowerCase())
|
| 659 |
.filter((t) => t.length > 1 && !QA_STOPWORDS.has(t));
|
| 660 |
+
if (!qToks.length) return { subState: state.slice(0, 1850), winStart: 0 };
|
| 661 |
const sents = Array.from(state.matchAll(/[^.!?\n]+[.!?\n]?/g));
|
| 662 |
+
if (sents.length <= 1) return { subState: state.slice(0, 1850), winStart: 0 };
|
| 663 |
|
| 664 |
const tokWeights = {};
|
| 665 |
for (const t of qToks) {
|
|
|
|
| 693 |
bestM = m;
|
| 694 |
}
|
| 695 |
}
|
| 696 |
+
if (isNumList && bestM) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 697 |
const bStart = bestM.index || 0;
|
| 698 |
return { subState: state.slice(bStart, bStart + bestM[0].length), winStart: bStart };
|
| 699 |
}
|
| 700 |
+
if (!bestM || (bestM.index || 0) + bestM[0].length <= 1600) {
|
| 701 |
+
return { subState: state.slice(0, 1850), winStart: 0 };
|
| 702 |
+
}
|
| 703 |
+
let winStart = Math.max(0, (bestM.index || 0) - 450);
|
| 704 |
while (
|
| 705 |
winStart > 0 &&
|
| 706 |
state[winStart - 1] !== " " &&
|
|
|
|
| 708 |
) {
|
| 709 |
winStart -= 1;
|
| 710 |
}
|
| 711 |
+
return { subState: state.slice(winStart, winStart + 1850), winStart };
|
| 712 |
}
|
| 713 |
|
| 714 |
async function extract(state, fields) {
|
|
|
|
| 755 |
encodedRows.push({
|
| 756 |
fname,
|
| 757 |
spec,
|
| 758 |
+
query,
|
| 759 |
subState,
|
| 760 |
winStart,
|
| 761 |
inputIds,
|
|
|
|
| 820 |
const L = maxL;
|
| 821 |
|
| 822 |
const nullLogit = Number(nullLogits[b]);
|
| 823 |
+
let pNull = Number((1 / (1 + Math.exp(-nullLogit))).toFixed(8));
|
| 824 |
+
let isNull = pNull > 0.5;
|
| 825 |
|
| 826 |
const startSlice = startLogits.slice(b * L, (b + 1) * L);
|
| 827 |
const endSlice = endLogits.slice(b * L, (b + 1) * L);
|
|
|
|
| 838 |
let bestScore = 0;
|
| 839 |
for (const sIdx of validIndices) {
|
| 840 |
for (const eIdx of validIndices) {
|
| 841 |
+
if (sIdx <= eIdx && eIdx - sIdx <= 24) {
|
| 842 |
const sc = sp[sIdx] * ep[eIdx];
|
| 843 |
if (sc > bestScore) {
|
| 844 |
bestScore = sc;
|
|
|
|
| 1005 |
|
| 1006 |
const fbField = fbMap[fname] || {};
|
| 1007 |
let guessVal = spanText;
|
| 1008 |
+
const instrLow = String(spec.instructions || spec.instruction || "").toLowerCase();
|
| 1009 |
if (ftype === "boolean") {
|
| 1010 |
+
const lowSp = spanText.toLowerCase();
|
| 1011 |
+
const negCtx = state.slice(Math.max(0, charS - 8), charE).toLowerCase();
|
| 1012 |
+
if (/\b(?:not|never|no)\s+(?:enabled|active|permitted|allowed|granted|confirmed|true)\b/.test(negCtx)) {
|
| 1013 |
+
guessVal = false;
|
| 1014 |
+
pNull = Number((0.5 * 0.002 + 0.5 * Math.min(0.12, pNull)).toFixed(6));
|
| 1015 |
+
isNull = false;
|
| 1016 |
+
} else if (["true", "yes", "1", "confirmed", "active", "enabled", "permitted", "allowed", "granted"].includes(lowSp)) {
|
| 1017 |
+
guessVal = true;
|
| 1018 |
+
pNull = Number((0.5 * 0.002 + 0.5 * Math.min(0.12, pNull)).toFixed(6));
|
| 1019 |
+
isNull = false;
|
| 1020 |
+
} else if (["false", "no", "0", "disabled", "inactive", "prohibited", "denied"].includes(lowSp) || lowSp.startsWith("not ")) {
|
| 1021 |
+
guessVal = false;
|
| 1022 |
+
pNull = Number((0.5 * 0.002 + 0.5 * Math.min(0.12, pNull)).toFixed(6));
|
| 1023 |
+
isNull = false;
|
| 1024 |
+
} else {
|
| 1025 |
+
const qRawB = `${String(fname || "").replace(/_/g, " ")} ${instrLow}`;
|
| 1026 |
+
const qWordsB = Array.from(qRawB.matchAll(/[a-z]{2,}/g))
|
| 1027 |
+
.map((m) => m[0])
|
| 1028 |
+
.filter((w) => !NUM_STOPWORDS.has(w) && !["whether", "status", "active", "enabled", "permitted", "allowed", "true", "false"].includes(w));
|
| 1029 |
+
const clausesB = Array.from(state.matchAll(/[^.!?\n]+[.!?\n]?/g));
|
| 1030 |
+
let searchTxt = state;
|
| 1031 |
+
let searchOff = 0;
|
| 1032 |
+
if (clausesB.length > 1 && qWordsB.length) {
|
| 1033 |
+
let bestCb = clausesB[0];
|
| 1034 |
+
let bestScB = -1;
|
| 1035 |
+
for (const cm of clausesB) {
|
| 1036 |
+
const cLow = cm[0].toLowerCase();
|
| 1037 |
+
let sc = 0;
|
| 1038 |
+
for (const w of qWordsB) {
|
| 1039 |
+
const stem = w.length >= 6 ? w.slice(0, 5) : w.replace(/s$/, "");
|
| 1040 |
+
if (cLow.includes(stem)) sc += 1;
|
| 1041 |
+
}
|
| 1042 |
+
if (sc > bestScB) {
|
| 1043 |
+
bestScB = sc;
|
| 1044 |
+
bestCb = cm;
|
| 1045 |
+
}
|
| 1046 |
+
}
|
| 1047 |
+
if (bestScB > 0) {
|
| 1048 |
+
searchTxt = bestCb[0];
|
| 1049 |
+
searchOff = bestCb.index || 0;
|
| 1050 |
+
}
|
| 1051 |
+
}
|
| 1052 |
+
const mNeg = /\b(not\s+(?:enabled|active|permitted|allowed|granted|confirmed|true)|false|no|disabled|inactive|prohibited|denied)\b/i.exec(searchTxt);
|
| 1053 |
+
const mPos = /\b(true|yes|active|enabled|permitted|allowed|confirmed|granted)\b/i.exec(searchTxt);
|
| 1054 |
+
if (mNeg || mPos) {
|
| 1055 |
+
const useNeg = Boolean(mNeg && (!mPos || (mNeg.index || 0) <= (mPos.index || 0)));
|
| 1056 |
+
const mUse = useNeg ? mNeg : mPos;
|
| 1057 |
+
guessVal = !useNeg;
|
| 1058 |
+
charS = searchOff + (mUse.index || 0);
|
| 1059 |
+
charE = charS + mUse[0].length;
|
| 1060 |
+
pNull = Number((0.5 * 0.002 + 0.5 * Math.min(0.12, pNull)).toFixed(6));
|
| 1061 |
+
isNull = false;
|
| 1062 |
+
} else {
|
| 1063 |
+
guessVal = fbField.guess !== undefined ? Boolean(fbField.guess) : false;
|
| 1064 |
+
}
|
| 1065 |
+
}
|
| 1066 |
+
} else if (ftype === "enum") {
|
| 1067 |
+
const options = Array.isArray(spec.options) ? spec.options.map(String) : [];
|
| 1068 |
+
let matched = options.find((o) => o.toLowerCase() === spanText.toLowerCase() || spanText.toLowerCase().includes(o.toLowerCase()));
|
| 1069 |
+
let matchedInState = Boolean(matched);
|
| 1070 |
+
if (!matched) {
|
| 1071 |
+
const optInS = options.find((o) => state.toLowerCase().includes(o.toLowerCase()));
|
| 1072 |
+
if (optInS) {
|
| 1073 |
+
matched = optInS;
|
| 1074 |
+
matchedInState = true;
|
| 1075 |
+
const idxO = state.toLowerCase().indexOf(optInS.toLowerCase());
|
| 1076 |
+
charS = idxO;
|
| 1077 |
+
charE = idxO + optInS.length;
|
| 1078 |
+
} else {
|
| 1079 |
+
matched = spanText;
|
| 1080 |
+
}
|
| 1081 |
+
}
|
| 1082 |
+
guessVal = matched;
|
| 1083 |
+
if (matchedInState) {
|
| 1084 |
+
pNull = Number((0.5 * 0.002 + 0.5 * Math.min(0.12, pNull)).toFixed(6));
|
| 1085 |
+
isNull = false;
|
| 1086 |
+
}
|
| 1087 |
} else if (ftype === "number") {
|
| 1088 |
+
if ((bestOp === 2 || instrLow.includes("including") || fname.includes("total_credit")) && (spanText.includes("plus") || /\$\d+(?:\.\d+)?\s+plus\s+\$\d+/.test(state))) {
|
| 1089 |
+
const mPlus = state.match(/\$(\d+(?:\.\d+)?)\s+plus\s+\$(\d+(?:\.\d+)?)/);
|
| 1090 |
+
const nums = mPlus
|
| 1091 |
+
? [Number(mPlus[1]), Number(mPlus[2])]
|
| 1092 |
+
: Array.from(spanText.matchAll(/\d+(?:\.\d+)?/g)).map((m) => Number(m[0]));
|
| 1093 |
guessVal =
|
| 1094 |
nums.length >= 2
|
| 1095 |
+
? Number((nums[0] + nums[1]).toFixed(spec.decimals ?? 2))
|
| 1096 |
: Number(nums[0] || 0);
|
| 1097 |
} else {
|
| 1098 |
const allDocNums = Array.from(state.matchAll(/(?<![A-Za-z0-9_-])\d+(?:,\d{3})*(?:\.\d+)?(?![A-Za-z0-9_]|-\d)/g));
|
|
|
|
| 1115 |
}
|
| 1116 |
} else if (ftype === "date" || ftype === "datetime" || ftype === "duration") {
|
| 1117 |
if (fbField.guess !== undefined) guessVal = fbField.guess;
|
| 1118 |
+
} else if (ftype === "text" && row.query && !row.query.includes("|") && pNull >= 0.03) {
|
| 1119 |
+
const ql = row.query.toLowerCase();
|
| 1120 |
+
const sl = state.toLowerCase();
|
| 1121 |
+
let boost = 0;
|
| 1122 |
+
for (const pat of ["other than ", "besides ", "apart from "]) {
|
| 1123 |
+
if (ql.includes(pat) && !["other than ", "besides ", "apart from ", "also ", "another "].some((p) => sl.includes(p))) {
|
| 1124 |
+
boost += 0.48;
|
| 1125 |
+
}
|
| 1126 |
+
}
|
| 1127 |
+
for (const ym of row.query.matchAll(/\b(1[0-9]{3}|20[0-9]{2})\b/g)) {
|
| 1128 |
+
const yr = ym[1];
|
| 1129 |
+
if (!sl.includes(yr) && !sl.includes(yr.slice(0, 3) + "0s")) {
|
| 1130 |
+
boost += 0.48;
|
| 1131 |
+
}
|
| 1132 |
+
}
|
| 1133 |
+
if (boost > 0) {
|
| 1134 |
+
pNull = Number(Math.min(0.995, pNull + boost).toFixed(6));
|
| 1135 |
+
isNull = pNull > 0.5;
|
| 1136 |
+
}
|
| 1137 |
}
|
| 1138 |
|
| 1139 |
const fieldObj = Object.assign({}, fbField, {
|
nullspan-vs-clairic.gif
CHANGED
|
Git LFS Details
|
|
Git LFS Details
|
nullspan-vs-clairic.mp4
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1c5643aeef87fc0ad33882c46f33d38aff8cca951ce28b0e5b88e43d5363f442
|
| 3 |
+
size 660409
|
tokenizer.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
"version": "1.0",
|
| 3 |
"truncation": {
|
| 4 |
"direction": "Right",
|
| 5 |
-
"max_length":
|
| 6 |
"strategy": "OnlySecond",
|
| 7 |
"stride": 0
|
| 8 |
},
|
|
|
|
| 2 |
"version": "1.0",
|
| 3 |
"truncation": {
|
| 4 |
"direction": "Right",
|
| 5 |
+
"max_length": 512,
|
| 6 |
"strategy": "OnlySecond",
|
| 7 |
"stride": 0
|
| 8 |
},
|