| { |
| "repo_id": "gnumanth/nullspan", |
| "backbone": "ahotrod/electra_large_discriminator_squad2_512", |
| "params_m": 344.22, |
| "max_length": 512, |
| "device": "Tesla V100-SXM2-16GB", |
| "train_seconds": 1439.16, |
| "epochs": 8, |
| "train_samples": 8290, |
| "cal_dev_samples": 720, |
| "calibration": { |
| "null_bias_delta": 5.9, |
| "null_temperature": 0.7, |
| "span_temperature": 1.2, |
| "qa_null_bias": 0.0, |
| "qa_null_scale": 0.22 |
| }, |
| "val_metrics": { |
| "null_accuracy": 96.94, |
| "null_auroc": 0.99273, |
| "null_ece": 0.01553, |
| "null_brier_score": 0.02711, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 100.0, |
| "quantile_mae_norm": 0.0007, |
| "quantile_pinball_loss": 0.0001 |
| }, |
| "canonical_eval": { |
| "canonical_field_accuracy": 100.0, |
| "adversarial_stress_accuracy": 100.0, |
| "sealed_squad_v2_accuracy": 83.33, |
| "sealed_contracts_accuracy": 97.92, |
| "invariant_verification_accuracy": 100.0, |
| "mean_doc_latency_ms": 108.06, |
| "examples": { |
| "nullspan_incident_postmortem": { |
| "latency_ms": 171.14, |
| "num_fields": 10, |
| "invariants_passed": true, |
| "answers": { |
| "reporter": { |
| "type": "text", |
| "p_null": 0.0, |
| "span": [ |
| 276, |
| 280 |
| ], |
| "guess": "Mira", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "Mira" |
| }, |
| "incident_id": { |
| "type": "text", |
| "p_null": 0.0, |
| "span": [ |
| 9, |
| 18 |
| ], |
| "guess": "INC-40928", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "INC-40928" |
| }, |
| "opened_on": { |
| "type": "date", |
| "p_null": 0.0, |
| "span": [ |
| 29, |
| 41 |
| ], |
| "guess": "2026-04-12", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "2026-04-12" |
| }, |
| "total_credit": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 109, |
| 130 |
| ], |
| "guess": 1550.75, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 1550.75 |
| }, |
| "outage_days": { |
| "type": "duration", |
| "p_null": 0.0, |
| "unit": "days", |
| "span": [ |
| 29, |
| 41 |
| ], |
| "guess": 7, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 7 |
| }, |
| "failure_modes": { |
| "type": "list", |
| "value": [ |
| "502 Bad Gateway errors", |
| "TLS handshake timeouts" |
| ], |
| "spans": [ |
| [ |
| 49, |
| 71 |
| ], |
| [ |
| 78, |
| 100 |
| ] |
| ], |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0 |
| }, |
| "severity": { |
| "type": "enum", |
| "p_null": 0.0, |
| "span": [ |
| 155, |
| 160 |
| ], |
| "options": [ |
| "sev-1", |
| "sev-2", |
| "sev-3" |
| ], |
| "probs": { |
| "sev-1": 0.999999, |
| "sev-2": 1e-06, |
| "sev-3": 1e-06 |
| }, |
| "guess": "sev-1", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "sev-1" |
| }, |
| "data_loss": { |
| "type": "boolean", |
| "p_null": 0.0, |
| "p_true": 1e-06, |
| "span": [ |
| 188, |
| 193 |
| ], |
| "guess": false, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": false |
| }, |
| "cve_id": { |
| "type": "text", |
| "p_null": 1.0, |
| "span": [ |
| 109, |
| 117 |
| ], |
| "guess": "$1240.50", |
| "confidence": 0.999999, |
| "span_entropy_bits": 2.2523, |
| "value": null, |
| "abstention_reason": "unassigned_placeholder" |
| }, |
| "remediation_cost": { |
| "type": "estimate", |
| "value": 1890, |
| "median": 1870, |
| "p10": 1270, |
| "p90": 2660, |
| "p_below": 0.0, |
| "p_above": 0.0, |
| "confidence": 0.999999 |
| } |
| } |
| }, |
| "nullspan_gpu_invoice": { |
| "latency_ms": 129.52, |
| "num_fields": 9, |
| "invariants_passed": true, |
| "answers": { |
| "provider": { |
| "type": "text", |
| "p_null": 0.0, |
| "span": [ |
| 13, |
| 29 |
| ], |
| "guess": "TensorCore Cloud", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "TensorCore Cloud" |
| }, |
| "invoice_number": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 39, |
| 44 |
| ], |
| "guess": 88310, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 88310 |
| }, |
| "issued_at": { |
| "type": "datetime", |
| "p_null": 0.0, |
| "span": [ |
| 55, |
| 71 |
| ], |
| "guess": "2026-08-14T18:20:00", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "2026-08-14T18:20:00" |
| }, |
| "nodes": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 79, |
| 80 |
| ], |
| "guess": 4, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 4 |
| }, |
| "hourly_rate": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 84, |
| 88 |
| ], |
| "guess": 3.5, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 3.5 |
| }, |
| "subtotal": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 105, |
| 110 |
| ], |
| "guess": 14, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 14, |
| "verified": true |
| }, |
| "egress_fee": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 124, |
| 128 |
| ], |
| "guess": 0, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 0 |
| }, |
| "priority_surcharge": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 150, |
| 154 |
| ], |
| "guess": 1.75, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 1.75 |
| }, |
| "promo_rebate": { |
| "type": "number", |
| "p_null": 0.996, |
| "span": [ |
| 39, |
| 44 |
| ], |
| "guess": 88310, |
| "confidence": 0.996, |
| "span_entropy_bits": 0.18, |
| "abstention_reason": "missing_field", |
| "value": null |
| } |
| } |
| }, |
| "nullspan_cluster_comps": { |
| "latency_ms": 54.41, |
| "num_fields": 1, |
| "invariants_passed": true, |
| "answers": { |
| "tokens_per_sec": { |
| "type": "estimate", |
| "value": 23200, |
| "median": 23000, |
| "p10": 20400, |
| "p90": 26200, |
| "p_below": 0.0, |
| "p_above": 0.0, |
| "confidence": 0.999762 |
| } |
| } |
| }, |
| "nullspan_edge_forecast": { |
| "latency_ms": 77.16, |
| "num_fields": 1, |
| "invariants_passed": true, |
| "answers": { |
| "requests": { |
| "type": "forecast", |
| "value": [ |
| 2416983 |
| ], |
| "median": [ |
| 2419497 |
| ], |
| "p10": [ |
| 2262466 |
| ], |
| "p90": [ |
| 2560198 |
| ], |
| "confidence": 0.999999 |
| } |
| } |
| } |
| }, |
| "adversarial_examples": { |
| "adv_prompt_injection_and_decoy_id": { |
| "latency_ms": 86.74, |
| "num_fields": 4, |
| "invariants_passed": true, |
| "answers": { |
| "incident_id": { |
| "type": "text", |
| "p_null": 0.0, |
| "span": [ |
| 9, |
| 18 |
| ], |
| "guess": "INC-77401", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": "INC-77401" |
| }, |
| "cve_id": { |
| "type": "text", |
| "p_null": 1.0, |
| "span": [ |
| 103, |
| 110 |
| ], |
| "guess": "$900.00", |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.5024, |
| "value": null, |
| "abstention_reason": "unassigned_placeholder" |
| }, |
| "total_credit": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 103, |
| 123 |
| ], |
| "guess": 1080, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 1080 |
| }, |
| "data_loss": { |
| "type": "boolean", |
| "p_null": 0.0, |
| "p_true": 1e-06, |
| "span": [ |
| 181, |
| 186 |
| ], |
| "guess": false, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": false |
| } |
| } |
| }, |
| "adv_negated_rebate_and_invariant": { |
| "latency_ms": 61.82, |
| "num_fields": 4, |
| "invariants_passed": true, |
| "answers": { |
| "nodes": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 77, |
| 78 |
| ], |
| "guess": 6, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 6 |
| }, |
| "hourly_rate": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 82, |
| 87 |
| ], |
| "guess": 12.5, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 12.5 |
| }, |
| "subtotal": { |
| "type": "number", |
| "p_null": 0.0, |
| "span": [ |
| 104, |
| 109 |
| ], |
| "guess": 75, |
| "confidence": 0.999999, |
| "span_entropy_bits": 0.0, |
| "value": 75, |
| "verified": true |
| }, |
| "promo_rebate": { |
| "type": "number", |
| "p_null": 0.996, |
| "span": [ |
| 37, |
| 42 |
| ], |
| "guess": 55190, |
| "confidence": 0.996, |
| "span_entropy_bits": 0.18, |
| "abstention_reason": "missing_field", |
| "value": null |
| } |
| } |
| } |
| } |
| }, |
| "history": [ |
| { |
| "epoch": 1, |
| "train_loss": 5.062, |
| "composite_dev": 91.31, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.98847, |
| "null_ece": 0.02311, |
| "null_brier_score": 0.02888, |
| "span_exact_match": 86.23, |
| "op_accuracy": 82.92, |
| "list_bio_token_accuracy": 98.73, |
| "quantile_mae_norm": 0.0065, |
| "quantile_pinball_loss": 0.0013 |
| }, |
| { |
| "epoch": 2, |
| "train_loss": 0.9522, |
| "composite_dev": 94.01, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99043, |
| "null_ece": 0.02436, |
| "null_brier_score": 0.02882, |
| "span_exact_match": 87.82, |
| "op_accuracy": 96.67, |
| "list_bio_token_accuracy": 99.29, |
| "quantile_mae_norm": 0.002, |
| "quantile_pinball_loss": 0.0009 |
| }, |
| { |
| "epoch": 3, |
| "train_loss": 0.4912, |
| "composite_dev": 94.09, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99233, |
| "null_ece": 0.02481, |
| "null_brier_score": 0.02899, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 96.47, |
| "quantile_mae_norm": 0.0009, |
| "quantile_pinball_loss": 0.0002 |
| }, |
| { |
| "epoch": 4, |
| "train_loss": 0.3236, |
| "composite_dev": 94.2, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99127, |
| "null_ece": 0.0244, |
| "null_brier_score": 0.02882, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 97.18, |
| "quantile_mae_norm": 0.0013, |
| "quantile_pinball_loss": 0.0002 |
| }, |
| { |
| "epoch": 5, |
| "train_loss": 0.2512, |
| "composite_dev": 94.62, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99255, |
| "null_ece": 0.02432, |
| "null_brier_score": 0.02866, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 100.0, |
| "quantile_mae_norm": 0.0011, |
| "quantile_pinball_loss": 0.0002 |
| }, |
| { |
| "epoch": 6, |
| "train_loss": 0.1666, |
| "composite_dev": 94.62, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99245, |
| "null_ece": 0.02437, |
| "null_brier_score": 0.02876, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 100.0, |
| "quantile_mae_norm": 0.0007, |
| "quantile_pinball_loss": 0.0001 |
| }, |
| { |
| "epoch": 7, |
| "train_loss": 0.1201, |
| "composite_dev": 94.62, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99225, |
| "null_ece": 0.02439, |
| "null_brier_score": 0.0288, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 100.0, |
| "quantile_mae_norm": 0.0007, |
| "quantile_pinball_loss": 0.0001 |
| }, |
| { |
| "epoch": 8, |
| "train_loss": 0.0938, |
| "composite_dev": 94.62, |
| "null_accuracy": 96.81, |
| "null_auroc": 0.99204, |
| "null_ece": 0.02439, |
| "null_brier_score": 0.02881, |
| "span_exact_match": 87.82, |
| "op_accuracy": 100.0, |
| "list_bio_token_accuracy": 100.0, |
| "quantile_mae_norm": 0.0007, |
| "quantile_pinball_loss": 0.0001 |
| } |
| ] |
| } |
