File size: 5,114 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
{
  "task": "ledgar",
  "name": "ce-seed0",
  "seed": 0,
  "seconds": 25980.469927399987,
  "dev": {
    "rows": 2000,
    "seed": 20261118,
    "train_rows": 57934,
    "dropped_same_text": 66
  },
  "base": {
    "repo": "convaiinnovations/laya",
    "revision": "85cc74ddae6109077ebe2853a9b3077fb6606a78",
    "weights_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c"
  },
  "config": {
    "epochs": 4,
    "micro_batch": 8,
    "grad_accum": 8,
    "encoder_lr": 2.5e-05,
    "head_lr": 0.0001,
    "min_lr": 1e-06,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "loss": "soft-ce",
    "rl_samples": 4,
    "sigma_start": 0.4,
    "sigma_end": 0.1,
    "w_sph": 0.75,
    "w_rps": 1.0,
    "shuffle_options": [],
    "option_layout": null,
    "freeze_encoder": false,
    "calib_max": 400,
    "calib_frac": 0.1,
    "calib_seed": 20260922,
    "seed": 0,
    "max_len": 768,
    "head_max_len": 512,
    "amp": null,
    "gradient_checkpointing": null,
    "log_every": 100,
    "label_smoothing": 0.0,
    "text_column": "text",
    "label_column": "label",
    "question_id": "label",
    "instructions": null,
    "eval_data": null,
    "target_error": 0.01,
    "min_abstain_n": 10
  },
  "summary": {
    "train_items": 57534,
    "calibration_items": 400,
    "eval_items": 400,
    "skipped": {},
    "epoch_loss": [
      0.6323961586747381,
      0.32258454987853263,
      0.1312420500772953,
      0.04140990528605578
    ],
    "temperature": [
      1.9877519607543945,
      1.0,
      1.0
    ],
    "temperature_by_options": {},
    "n_by_bucket": {
      "choice:11+": 400
    },
    "calibration": {
      "choice": {
        "items": 400,
        "temperature": 1.9877519607543945,
        "issues": [],
        "accuracy": 0.8575,
        "mean_confidence": 0.8994799126217312
      },
      "score": {
        "items": 0,
        "temperature": 1.0,
        "issues": []
      },
      "noul": {
        "items": 0,
        "temperature": 1.0,
        "issues": []
      }
    },
    "abstention_thresholds": {
      "choice:11+": 1.0
    },
    "train_report": {
      "eval_source": "calibration_slice",
      "eval_mode": "in_sample_calibration",
      "is_held_out": false,
      "eval_items": 400,
      "note": "Metrics reflect in-sample calibration fit, not independent held-out evaluation. Provide --eval for held-out validation.",
      "before": {
        "items": 400,
        "loss": 23.8939,
        "accuracy": 0.3875,
        "mean_confidence": 0.9603,
        "ece": 0.5728,
        "brier": 1.1697,
        "brier_top1": 0.5598,
        "by_type": {
          "choice": {
            "items": 400,
            "loss": 23.8939,
            "accuracy": 0.3875,
            "mean_confidence": 0.9603,
            "ece": 0.5728,
            "brier": 1.1697,
            "brier_top1": 0.5598
          }
        }
      },
      "after": {
        "items": 400,
        "loss": 0.496,
        "accuracy": 0.8575,
        "mean_confidence": 0.8995,
        "ece": 0.0657,
        "brier": 0.2175,
        "brier_top1": 0.0981,
        "by_type": {
          "choice": {
            "items": 400,
            "loss": 0.496,
            "accuracy": 0.8575,
            "mean_confidence": 0.8995,
            "ece": 0.0657,
            "brier": 0.2175,
            "brier_top1": 0.0981
          }
        }
      },
      "comparison": {
        "delta_accuracy": 0.47,
        "delta_loss": -23.3979,
        "delta_ece": -0.5071,
        "delta_brier": -0.9522,
        "delta_brier_top1": -0.4617,
        "delta_mean_confidence": -0.0608
      },
      "calibration": {
        "choice": {
          "items": 400,
          "temperature": 1.9877519607543945,
          "issues": [],
          "accuracy": 0.8575,
          "mean_confidence": 0.8994799126217312
        },
        "score": {
          "items": 0,
          "temperature": 1.0,
          "issues": []
        },
        "noul": {
          "items": 0,
          "temperature": 1.0,
          "issues": []
        }
      },
      "abstention_thresholds": {
        "choice:11+": 1.0
      },
      "training": {
        "epochs": 4,
        "micro_batch": 8,
        "grad_accum": 8,
        "encoder_lr": 2.5e-05,
        "head_lr": 0.0001,
        "min_lr": 1e-06,
        "weight_decay": 0.01,
        "grad_clip": 1.0,
        "loss": "soft-ce",
        "rl_samples": 4,
        "sigma_start": 0.4,
        "sigma_end": 0.1,
        "w_sph": 0.75,
        "w_rps": 1.0,
        "shuffle_options": [],
        "option_layout": null,
        "freeze_encoder": false,
        "calib_max": 400,
        "calib_frac": 0.1,
        "calib_seed": 20260922,
        "seed": 0,
        "max_len": 768,
        "head_max_len": 512,
        "amp": null,
        "gradient_checkpointing": null,
        "log_every": 100,
        "label_smoothing": 0.0,
        "text_column": "text",
        "label_column": "label",
        "question_id": "label",
        "instructions": null,
        "eval_data": null,
        "target_error": 0.01,
        "min_abstain_n": 10
      }
    }
  }
}