File size: 3,999 Bytes
c03caac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
{
  "eval_source": "data\\sets\\val.jsonl",
  "eval_mode": "overlapping_eval",
  "is_held_out": false,
  "eval_items": 2902,
  "note": "Evaluation set contains inputs that overlap training data (7) and calibration data (1) (total 8/2902 items); metrics do not reflect strictly independent held-out evaluation.",
  "before": {
    "items": 2902,
    "loss": 2.5109,
    "accuracy": 0.4621,
    "mean_confidence": 0.8259,
    "ece": 0.3638,
    "brier": 0.8539,
    "brier_top1": 0.3883,
    "by_type": {
      "choice": {
        "items": 1083,
        "loss": 1.7383,
        "accuracy": 0.506,
        "mean_confidence": 0.7489,
        "ece": 0.2429,
        "brier": 0.7265,
        "brier_top1": 0.2916
      },
      "score": {
        "items": 347,
        "loss": 2.1853,
        "accuracy": 0.2478,
        "mean_confidence": 0.7083,
        "ece": 0.4605,
        "brier": 1.0195,
        "brier_top1": 0.4103
      },
      "noul": {
        "items": 1472,
        "loss": 3.1561,
        "accuracy": 0.4803,
        "mean_confidence": 0.9102,
        "ece": 0.4302,
        "brier": 0.9086,
        "brier_top1": 0.4543
      }
    }
  },
  "after": {
    "items": 2902,
    "loss": 0.1931,
    "accuracy": 0.9549,
    "mean_confidence": 0.9724,
    "ece": 0.0182,
    "brier": 0.0806,
    "brier_top1": 0.0389,
    "by_type": {
      "choice": {
        "items": 1083,
        "loss": 0.3187,
        "accuracy": 0.9261,
        "mean_confidence": 0.9599,
        "ece": 0.0343,
        "brier": 0.1304,
        "brier_top1": 0.0618
      },
      "score": {
        "items": 347,
        "loss": 0.2915,
        "accuracy": 0.9222,
        "mean_confidence": 0.9595,
        "ece": 0.0383,
        "brier": 0.1325,
        "brier_top1": 0.0651
      },
      "noul": {
        "items": 1472,
        "loss": 0.0776,
        "accuracy": 0.9837,
        "mean_confidence": 0.9847,
        "ece": 0.0021,
        "brier": 0.0316,
        "brier_top1": 0.0158
      }
    }
  },
  "comparison": {
    "delta_accuracy": 0.4928,
    "delta_loss": -2.3178,
    "delta_ece": -0.3456,
    "delta_brier": -0.7733,
    "delta_brier_top1": -0.3494,
    "delta_mean_confidence": 0.1465
  },
  "calibration": {
    "choice": {
      "items": 126,
      "temperature": 4.242581367492676,
      "issues": [],
      "accuracy": 0.9523809523809523,
      "mean_confidence": 0.954239701485137
    },
    "score": {
      "items": 29,
      "temperature": 4.761922359466553,
      "issues": [
        "fitted on only 29 calibration items"
      ],
      "accuracy": 0.9655172413793104,
      "mean_confidence": 0.9647458537741626
    },
    "noul": {
      "items": 245,
      "temperature": 4.510099411010742,
      "issues": [],
      "accuracy": 0.9836734693877551,
      "mean_confidence": 0.9860408716061371
    }
  },
  "abstention_thresholds": {
    "noul:2": 0.946,
    "choice:6-10": 0.9404,
    "choice:2": 0.9841,
    "choice:3-5": 0.9557,
    "score:3-5": 0.9595
  },
  "training": {
    "epochs": 5,
    "micro_batch": 2,
    "grad_accum": 8,
    "encoder_lr": 2.5e-05,
    "head_lr": 0.0001,
    "min_lr": 1e-06,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "loss": "rlcd",
    "rl_samples": 4,
    "sigma_start": 0.4,
    "sigma_end": 0.1,
    "w_sph": 0.75,
    "w_rps": 1.0,
    "shuffle_options": [
      "choice"
    ],
    "option_layout": null,
    "freeze_encoder": false,
    "calib_max": 400,
    "calib_frac": 0.1,
    "calib_seed": 20260922,
    "seed": 0,
    "max_len": 2048,
    "head_max_len": null,
    "amp": null,
    "gradient_checkpointing": null,
    "log_every": 100,
    "label_smoothing": 0.0,
    "text_column": "text",
    "label_column": "label",
    "question_id": "label",
    "instructions": null,
    "eval_data": "data\\sets\\val.jsonl",
    "target_error": 0.1,
    "min_abstain_n": 10
  }
}