{
  "repo_id": "gnumanth/nullspan",
  "backbone": "ahotrod/electra_large_discriminator_squad2_512",
  "params_m": 344.22,
  "max_length": 512,
  "device": "Tesla V100-SXM2-16GB",
  "train_seconds": 1439.16,
  "epochs": 8,
  "train_samples": 8290,
  "cal_dev_samples": 720,
  "calibration": {
    "null_bias_delta": 5.9,
    "null_temperature": 0.7,
    "span_temperature": 1.2,
    "qa_null_bias": 0.0,
    "qa_null_scale": 0.22
  },
  "val_metrics": {
    "null_accuracy": 96.94,
    "null_auroc": 0.99273,
    "null_ece": 0.01553,
    "null_brier_score": 0.02711,
    "span_exact_match": 87.82,
    "op_accuracy": 100.0,
    "list_bio_token_accuracy": 100.0,
    "quantile_mae_norm": 0.0007,
    "quantile_pinball_loss": 0.0001
  },
  "canonical_eval": {
    "canonical_field_accuracy": 100.0,
    "adversarial_stress_accuracy": 100.0,
    "sealed_squad_v2_accuracy": 83.33,
    "sealed_contracts_accuracy": 97.92,
    "invariant_verification_accuracy": 100.0,
    "mean_doc_latency_ms": 108.06,
    "examples": {
      "nullspan_incident_postmortem": {
        "latency_ms": 171.14,
        "num_fields": 10,
        "invariants_passed": true,
        "answers": {
          "reporter": {
            "type": "text",
            "p_null": 0.0,
            "span": [
              276,
              280
            ],
            "guess": "Mira",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "Mira"
          },
          "incident_id": {
            "type": "text",
            "p_null": 0.0,
            "span": [
              9,
              18
            ],
            "guess": "INC-40928",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "INC-40928"
          },
          "opened_on": {
            "type": "date",
            "p_null": 0.0,
            "span": [
              29,
              41
            ],
            "guess": "2026-04-12",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "2026-04-12"
          },
          "total_credit": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              109,
              130
            ],
            "guess": 1550.75,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 1550.75
          },
          "outage_days": {
            "type": "duration",
            "p_null": 0.0,
            "unit": "days",
            "span": [
              29,
              41
            ],
            "guess": 7,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 7
          },
          "failure_modes": {
            "type": "list",
            "value": [
              "502 Bad Gateway errors",
              "TLS handshake timeouts"
            ],
            "spans": [
              [
                49,
                71
              ],
              [
                78,
                100
              ]
            ],
            "confidence": 0.999999,
            "span_entropy_bits": 0.0
          },
          "severity": {
            "type": "enum",
            "p_null": 0.0,
            "span": [
              155,
              160
            ],
            "options": [
              "sev-1",
              "sev-2",
              "sev-3"
            ],
            "probs": {
              "sev-1": 0.999999,
              "sev-2": 1e-06,
              "sev-3": 1e-06
            },
            "guess": "sev-1",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "sev-1"
          },
          "data_loss": {
            "type": "boolean",
            "p_null": 0.0,
            "p_true": 1e-06,
            "span": [
              188,
              193
            ],
            "guess": false,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": false
          },
          "cve_id": {
            "type": "text",
            "p_null": 1.0,
            "span": [
              109,
              117
            ],
            "guess": "$1240.50",
            "confidence": 0.999999,
            "span_entropy_bits": 2.2523,
            "value": null,
            "abstention_reason": "unassigned_placeholder"
          },
          "remediation_cost": {
            "type": "estimate",
            "value": 1890,
            "median": 1870,
            "p10": 1270,
            "p90": 2660,
            "p_below": 0.0,
            "p_above": 0.0,
            "confidence": 0.999999
          }
        }
      },
      "nullspan_gpu_invoice": {
        "latency_ms": 129.52,
        "num_fields": 9,
        "invariants_passed": true,
        "answers": {
          "provider": {
            "type": "text",
            "p_null": 0.0,
            "span": [
              13,
              29
            ],
            "guess": "TensorCore Cloud",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "TensorCore Cloud"
          },
          "invoice_number": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              39,
              44
            ],
            "guess": 88310,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 88310
          },
          "issued_at": {
            "type": "datetime",
            "p_null": 0.0,
            "span": [
              55,
              71
            ],
            "guess": "2026-08-14T18:20:00",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "2026-08-14T18:20:00"
          },
          "nodes": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              79,
              80
            ],
            "guess": 4,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 4
          },
          "hourly_rate": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              84,
              88
            ],
            "guess": 3.5,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 3.5
          },
          "subtotal": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              105,
              110
            ],
            "guess": 14,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 14,
            "verified": true
          },
          "egress_fee": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              124,
              128
            ],
            "guess": 0,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 0
          },
          "priority_surcharge": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              150,
              154
            ],
            "guess": 1.75,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 1.75
          },
          "promo_rebate": {
            "type": "number",
            "p_null": 0.996,
            "span": [
              39,
              44
            ],
            "guess": 88310,
            "confidence": 0.996,
            "span_entropy_bits": 0.18,
            "abstention_reason": "missing_field",
            "value": null
          }
        }
      },
      "nullspan_cluster_comps": {
        "latency_ms": 54.41,
        "num_fields": 1,
        "invariants_passed": true,
        "answers": {
          "tokens_per_sec": {
            "type": "estimate",
            "value": 23200,
            "median": 23000,
            "p10": 20400,
            "p90": 26200,
            "p_below": 0.0,
            "p_above": 0.0,
            "confidence": 0.999762
          }
        }
      },
      "nullspan_edge_forecast": {
        "latency_ms": 77.16,
        "num_fields": 1,
        "invariants_passed": true,
        "answers": {
          "requests": {
            "type": "forecast",
            "value": [
              2416983
            ],
            "median": [
              2419497
            ],
            "p10": [
              2262466
            ],
            "p90": [
              2560198
            ],
            "confidence": 0.999999
          }
        }
      }
    },
    "adversarial_examples": {
      "adv_prompt_injection_and_decoy_id": {
        "latency_ms": 86.74,
        "num_fields": 4,
        "invariants_passed": true,
        "answers": {
          "incident_id": {
            "type": "text",
            "p_null": 0.0,
            "span": [
              9,
              18
            ],
            "guess": "INC-77401",
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": "INC-77401"
          },
          "cve_id": {
            "type": "text",
            "p_null": 1.0,
            "span": [
              103,
              110
            ],
            "guess": "$900.00",
            "confidence": 0.999999,
            "span_entropy_bits": 0.5024,
            "value": null,
            "abstention_reason": "unassigned_placeholder"
          },
          "total_credit": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              103,
              123
            ],
            "guess": 1080,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 1080
          },
          "data_loss": {
            "type": "boolean",
            "p_null": 0.0,
            "p_true": 1e-06,
            "span": [
              181,
              186
            ],
            "guess": false,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": false
          }
        }
      },
      "adv_negated_rebate_and_invariant": {
        "latency_ms": 61.82,
        "num_fields": 4,
        "invariants_passed": true,
        "answers": {
          "nodes": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              77,
              78
            ],
            "guess": 6,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 6
          },
          "hourly_rate": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              82,
              87
            ],
            "guess": 12.5,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 12.5
          },
          "subtotal": {
            "type": "number",
            "p_null": 0.0,
            "span": [
              104,
              109
            ],
            "guess": 75,
            "confidence": 0.999999,
            "span_entropy_bits": 0.0,
            "value": 75,
            "verified": true
          },
          "promo_rebate": {
            "type": "number",
            "p_null": 0.996,
            "span": [
              37,
              42
            ],
            "guess": 55190,
            "confidence": 0.996,
            "span_entropy_bits": 0.18,
            "abstention_reason": "missing_field",
            "value": null
          }
        }
      }
    }
  },
  "history": [
    {
      "epoch": 1,
      "train_loss": 5.062,
      "composite_dev": 91.31,
      "null_accuracy": 96.81,
      "null_auroc": 0.98847,
      "null_ece": 0.02311,
      "null_brier_score": 0.02888,
      "span_exact_match": 86.23,
      "op_accuracy": 82.92,
      "list_bio_token_accuracy": 98.73,
      "quantile_mae_norm": 0.0065,
      "quantile_pinball_loss": 0.0013
    },
    {
      "epoch": 2,
      "train_loss": 0.9522,
      "composite_dev": 94.01,
      "null_accuracy": 96.81,
      "null_auroc": 0.99043,
      "null_ece": 0.02436,
      "null_brier_score": 0.02882,
      "span_exact_match": 87.82,
      "op_accuracy": 96.67,
      "list_bio_token_accuracy": 99.29,
      "quantile_mae_norm": 0.002,
      "quantile_pinball_loss": 0.0009
    },
    {
      "epoch": 3,
      "train_loss": 0.4912,
      "composite_dev": 94.09,
      "null_accuracy": 96.81,
      "null_auroc": 0.99233,
      "null_ece": 0.02481,
      "null_brier_score": 0.02899,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 96.47,
      "quantile_mae_norm": 0.0009,
      "quantile_pinball_loss": 0.0002
    },
    {
      "epoch": 4,
      "train_loss": 0.3236,
      "composite_dev": 94.2,
      "null_accuracy": 96.81,
      "null_auroc": 0.99127,
      "null_ece": 0.0244,
      "null_brier_score": 0.02882,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 97.18,
      "quantile_mae_norm": 0.0013,
      "quantile_pinball_loss": 0.0002
    },
    {
      "epoch": 5,
      "train_loss": 0.2512,
      "composite_dev": 94.62,
      "null_accuracy": 96.81,
      "null_auroc": 0.99255,
      "null_ece": 0.02432,
      "null_brier_score": 0.02866,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 100.0,
      "quantile_mae_norm": 0.0011,
      "quantile_pinball_loss": 0.0002
    },
    {
      "epoch": 6,
      "train_loss": 0.1666,
      "composite_dev": 94.62,
      "null_accuracy": 96.81,
      "null_auroc": 0.99245,
      "null_ece": 0.02437,
      "null_brier_score": 0.02876,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 100.0,
      "quantile_mae_norm": 0.0007,
      "quantile_pinball_loss": 0.0001
    },
    {
      "epoch": 7,
      "train_loss": 0.1201,
      "composite_dev": 94.62,
      "null_accuracy": 96.81,
      "null_auroc": 0.99225,
      "null_ece": 0.02439,
      "null_brier_score": 0.0288,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 100.0,
      "quantile_mae_norm": 0.0007,
      "quantile_pinball_loss": 0.0001
    },
    {
      "epoch": 8,
      "train_loss": 0.0938,
      "composite_dev": 94.62,
      "null_accuracy": 96.81,
      "null_auroc": 0.99204,
      "null_ece": 0.02439,
      "null_brier_score": 0.02881,
      "span_exact_match": 87.82,
      "op_accuracy": 100.0,
      "list_bio_token_accuracy": 100.0,
      "quantile_mae_norm": 0.0007,
      "quantile_pinball_loss": 0.0001
    }
  ]
}