| { |
| "eval_source": "calibration_slice", |
| "eval_mode": "in_sample_calibration", |
| "is_held_out": false, |
| "eval_items": 400, |
| "note": "Metrics reflect in-sample calibration fit, not independent held-out evaluation. Provide --eval for held-out validation.", |
| "before": { |
| "items": 400, |
| "loss": 1.5651, |
| "accuracy": 0.56, |
| "mean_confidence": 0.8108, |
| "ece": 0.2508, |
| "brier": 0.6589, |
| "brier_top1": 0.2824, |
| "by_type": { |
| "choice": { |
| "items": 136, |
| "loss": 1.2561, |
| "accuracy": 0.7868, |
| "mean_confidence": 0.898, |
| "ece": 0.1288, |
| "brier": 0.3576, |
| "brier_top1": 0.1483 |
| }, |
| "score": { |
| "items": 129, |
| "loss": 2.1186, |
| "accuracy": 0.3101, |
| "mean_confidence": 0.6873, |
| "ece": 0.3772, |
| "brier": 0.9646, |
| "brier_top1": 0.3685 |
| }, |
| "noul": { |
| "items": 135, |
| "loss": 1.3475, |
| "accuracy": 0.5704, |
| "mean_confidence": 0.8411, |
| "ece": 0.2768, |
| "brier": 0.6702, |
| "brier_top1": 0.3351 |
| } |
| } |
| }, |
| "after": { |
| "items": 400, |
| "loss": 0.0509, |
| "accuracy": 0.985, |
| "mean_confidence": 0.9841, |
| "ece": 0.0104, |
| "brier": 0.0242, |
| "brier_top1": 0.0119, |
| "by_type": { |
| "choice": { |
| "items": 136, |
| "loss": 0.0, |
| "accuracy": 1.0, |
| "mean_confidence": 1.0, |
| "ece": 0.0, |
| "brier": 0.0, |
| "brier_top1": 0.0 |
| }, |
| "score": { |
| "items": 129, |
| "loss": 0.0468, |
| "accuracy": 0.9845, |
| "mean_confidence": 0.985, |
| "ece": 0.0107, |
| "brier": 0.0247, |
| "brier_top1": 0.0119 |
| }, |
| "noul": { |
| "items": 135, |
| "loss": 0.1061, |
| "accuracy": 0.9704, |
| "mean_confidence": 0.9671, |
| "ece": 0.0218, |
| "brier": 0.048, |
| "brier_top1": 0.024 |
| } |
| } |
| }, |
| "comparison": { |
| "delta_accuracy": 0.425, |
| "delta_loss": -1.5142, |
| "delta_ece": -0.2404, |
| "delta_brier": -0.6347, |
| "delta_brier_top1": -0.2705, |
| "delta_mean_confidence": 0.1733 |
| }, |
| "calibration": { |
| "choice": { |
| "items": 136, |
| "temperature": 0.9999958276748657, |
| "issues": [ |
| "the fit came back unchanged at 1.0 (accuracy 1.00, mean confidence 1.000 on the calibration items), which usually means they gave it nothing to correct; this type's confidences are uncalibrated" |
| ], |
| "accuracy": 1.0, |
| "mean_confidence": 0.9999959610103946 |
| }, |
| "score": { |
| "items": 129, |
| "temperature": 1.8285092115402222, |
| "issues": [], |
| "accuracy": 0.9844961240310077, |
| "mean_confidence": 0.9849781946721331 |
| }, |
| "noul": { |
| "items": 135, |
| "temperature": 3.370203733444214, |
| "issues": [], |
| "accuracy": 0.9703703703703703, |
| "mean_confidence": 0.9671178420842376 |
| } |
| }, |
| "abstention_thresholds": { |
| "choice:6-10": 0.9998, |
| "score:3-5": 0.5209, |
| "noul:2": 0.5493 |
| }, |
| "training": { |
| "epochs": 4, |
| "micro_batch": 2, |
| "grad_accum": 8, |
| "encoder_lr": 2.5e-05, |
| "head_lr": 0.0001, |
| "min_lr": 1e-06, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "loss": "soft-ce", |
| "rl_samples": 4, |
| "sigma_start": 0.4, |
| "sigma_end": 0.1, |
| "w_sph": 0.75, |
| "w_rps": 1.0, |
| "shuffle_options": [ |
| "choice" |
| ], |
| "option_layout": null, |
| "freeze_encoder": false, |
| "calib_max": 400, |
| "calib_frac": 0.1, |
| "calib_seed": 20260922, |
| "seed": 0, |
| "max_len": null, |
| "head_max_len": null, |
| "amp": null, |
| "gradient_checkpointing": null, |
| "log_every": 100, |
| "label_smoothing": 0.0, |
| "text_column": "text", |
| "label_column": "label", |
| "question_id": "label", |
| "instructions": null, |
| "eval_data": null, |
| "target_error": 0.1, |
| "min_abstain_n": 10 |
| } |
| } |
