{
  "eval_source": "calibration_slice",
  "eval_mode": "in_sample_calibration",
  "is_held_out": false,
  "eval_items": 400,
  "note": "Metrics reflect in-sample calibration fit, not independent held-out evaluation. Provide --eval for held-out validation.",
  "before": {
    "items": 400,
    "loss": 1.5651,
    "accuracy": 0.56,
    "mean_confidence": 0.8108,
    "ece": 0.2508,
    "brier": 0.6589,
    "brier_top1": 0.2824,
    "by_type": {
      "choice": {
        "items": 136,
        "loss": 1.2561,
        "accuracy": 0.7868,
        "mean_confidence": 0.898,
        "ece": 0.1288,
        "brier": 0.3576,
        "brier_top1": 0.1483
      },
      "score": {
        "items": 129,
        "loss": 2.1186,
        "accuracy": 0.3101,
        "mean_confidence": 0.6873,
        "ece": 0.3772,
        "brier": 0.9646,
        "brier_top1": 0.3685
      },
      "noul": {
        "items": 135,
        "loss": 1.3475,
        "accuracy": 0.5704,
        "mean_confidence": 0.8411,
        "ece": 0.2768,
        "brier": 0.6702,
        "brier_top1": 0.3351
      }
    }
  },
  "after": {
    "items": 400,
    "loss": 0.0509,
    "accuracy": 0.985,
    "mean_confidence": 0.9841,
    "ece": 0.0104,
    "brier": 0.0242,
    "brier_top1": 0.0119,
    "by_type": {
      "choice": {
        "items": 136,
        "loss": 0.0,
        "accuracy": 1.0,
        "mean_confidence": 1.0,
        "ece": 0.0,
        "brier": 0.0,
        "brier_top1": 0.0
      },
      "score": {
        "items": 129,
        "loss": 0.0468,
        "accuracy": 0.9845,
        "mean_confidence": 0.985,
        "ece": 0.0107,
        "brier": 0.0247,
        "brier_top1": 0.0119
      },
      "noul": {
        "items": 135,
        "loss": 0.1061,
        "accuracy": 0.9704,
        "mean_confidence": 0.9671,
        "ece": 0.0218,
        "brier": 0.048,
        "brier_top1": 0.024
      }
    }
  },
  "comparison": {
    "delta_accuracy": 0.425,
    "delta_loss": -1.5142,
    "delta_ece": -0.2404,
    "delta_brier": -0.6347,
    "delta_brier_top1": -0.2705,
    "delta_mean_confidence": 0.1733
  },
  "calibration": {
    "choice": {
      "items": 136,
      "temperature": 0.9999958276748657,
      "issues": [
        "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated"
      ],
      "accuracy": 1.0,
      "mean_confidence": 0.9999959610103946
    },
    "score": {
      "items": 129,
      "temperature": 1.8285092115402222,
      "issues": [],
      "accuracy": 0.9844961240310077,
      "mean_confidence": 0.9849781946721331
    },
    "noul": {
      "items": 135,
      "temperature": 3.370203733444214,
      "issues": [],
      "accuracy": 0.9703703703703703,
      "mean_confidence": 0.9671178420842376
    }
  },
  "abstention_thresholds": {
    "choice:6-10": 0.9998,
    "score:3-5": 0.5209,
    "noul:2": 0.5493
  },
  "training": {
    "epochs": 4,
    "micro_batch": 2,
    "grad_accum": 8,
    "encoder_lr": 2.5e-05,
    "head_lr": 0.0001,
    "min_lr": 1e-06,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "loss": "soft-ce",
    "rl_samples": 4,
    "sigma_start": 0.4,
    "sigma_end": 0.1,
    "w_sph": 0.75,
    "w_rps": 1.0,
    "shuffle_options": [
      "choice"
    ],
    "option_layout": null,
    "freeze_encoder": false,
    "calib_max": 400,
    "calib_frac": 0.1,
    "calib_seed": 20260922,
    "seed": 0,
    "max_len": null,
    "head_max_len": null,
    "amp": null,
    "gradient_checkpointing": null,
    "log_every": 100,
    "label_smoothing": 0.0,
    "text_column": "text",
    "label_column": "label",
    "question_id": "label",
    "instructions": null,
    "eval_data": null,
    "target_error": 0.1,
    "min_abstain_n": 10
  }
}