arnav95600 commited on
Commit
dc9d7ef
·
verified ·
1 Parent(s): b113052

simple-credit-r3 a/skeptical: final adapter (night 5)

Browse files
simple-credit-r3/a/skeptical/adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 32,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 16,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "down_proj",
29
+ "k_proj",
30
+ "gate_proj",
31
+ "q_proj",
32
+ "v_proj",
33
+ "up_proj",
34
+ "o_proj"
35
+ ],
36
+ "target_parameters": null,
37
+ "task_type": "CAUSAL_LM",
38
+ "trainable_token_indices": null,
39
+ "use_dora": false,
40
+ "use_qalora": false,
41
+ "use_rslora": false
42
+ }
simple-credit-r3/a/skeptical/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9a9c43b6726095f4e76fc816e8beb4a10ededfee47ccf435dfe236194b70aff
3
+ size 73911112
simple-credit-r3/a/skeptical/job.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model": "Qwen/Qwen2.5-1.5B-Instruct",
3
+ "init_adapter": "data/runs/simple-credit-r3/adapters/a/skeptical/cycle_4",
4
+ "dataset": "data/runs/simple-credit-r3/adapters/a/skeptical/cycle_5_pairs.jsonl",
5
+ "output_dir": "data/runs/simple-credit-r3/adapters/a/skeptical/cycle_5",
6
+ "seed": 186853418,
7
+ "lora_rank": 16,
8
+ "lora_alpha": 32,
9
+ "lora_dropout": 0.05,
10
+ "learning_rate": 0.00002,
11
+ "beta": 0.1,
12
+ "nll_weight": 1.0,
13
+ "epochs": 3,
14
+ "batch_size": 1,
15
+ "grad_accumulation": 2,
16
+ "max_length": 3072,
17
+ "gradient_checkpointing": true
18
+ }
simple-credit-r3/a/skeptical/train_metrics.json ADDED
@@ -0,0 +1,683 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "loss": 0.9391,
4
+ "grad_norm": 10.138032913208008,
5
+ "learning_rate": 2e-05,
6
+ "rewards/chosen": 20.699092864990234,
7
+ "rewards/rejected": 10.768796920776367,
8
+ "rewards/accuracies": 1.0,
9
+ "rewards/margins": 9.930295944213867,
10
+ "logps/chosen": -224.0374755859375,
11
+ "logps/rejected": -159.9487762451172,
12
+ "logits/chosen": -1.1399765014648438,
13
+ "logits/rejected": -1.0848315954208374,
14
+ "nll_loss": 0.7495715618133545,
15
+ "epoch": 0.07407407407407407,
16
+ "step": 1
17
+ },
18
+ {
19
+ "loss": 0.5462,
20
+ "grad_norm": 8.002249717712402,
21
+ "learning_rate": 1.9523809523809524e-05,
22
+ "rewards/chosen": 6.724698066711426,
23
+ "rewards/rejected": 1.2948265075683594,
24
+ "rewards/accuracies": 1.0,
25
+ "rewards/margins": 5.429871559143066,
26
+ "logps/chosen": -52.21371078491211,
27
+ "logps/rejected": -100.41464233398438,
28
+ "logits/chosen": -1.6014512777328491,
29
+ "logits/rejected": -1.810758113861084,
30
+ "nll_loss": 0.35883814096450806,
31
+ "epoch": 0.14814814814814814,
32
+ "step": 2
33
+ },
34
+ {
35
+ "loss": 0.8869,
36
+ "grad_norm": 13.98072624206543,
37
+ "learning_rate": 1.904761904761905e-05,
38
+ "rewards/chosen": 7.7275800704956055,
39
+ "rewards/rejected": 5.831037521362305,
40
+ "rewards/accuracies": 0.5,
41
+ "rewards/margins": 1.8965423107147217,
42
+ "logps/chosen": -53.18342208862305,
43
+ "logps/rejected": -54.69065856933594,
44
+ "logits/chosen": -1.2771055698394775,
45
+ "logits/rejected": -1.0625889301300049,
46
+ "nll_loss": 0.2501956820487976,
47
+ "epoch": 0.2222222222222222,
48
+ "step": 3
49
+ },
50
+ {
51
+ "loss": 0.3995,
52
+ "grad_norm": 2.0633723735809326,
53
+ "learning_rate": 1.8571428571428575e-05,
54
+ "rewards/chosen": 6.024013042449951,
55
+ "rewards/rejected": -0.4116058647632599,
56
+ "rewards/accuracies": 1.0,
57
+ "rewards/margins": 6.435618877410889,
58
+ "logps/chosen": -84.92457580566406,
59
+ "logps/rejected": -135.634521484375,
60
+ "logits/chosen": -1.4146201610565186,
61
+ "logits/rejected": -0.9178286790847778,
62
+ "nll_loss": 0.359701931476593,
63
+ "epoch": 0.2962962962962963,
64
+ "step": 4
65
+ },
66
+ {
67
+ "loss": 0.7378,
68
+ "grad_norm": 17.207197189331055,
69
+ "learning_rate": 1.8095238095238097e-05,
70
+ "rewards/chosen": 7.695709228515625,
71
+ "rewards/rejected": 3.7487001419067383,
72
+ "rewards/accuracies": 1.0,
73
+ "rewards/margins": 3.9470090866088867,
74
+ "logps/chosen": -63.10902404785156,
75
+ "logps/rejected": -212.8251953125,
76
+ "logits/chosen": -1.377525806427002,
77
+ "logits/rejected": -1.0982531309127808,
78
+ "nll_loss": 0.571829080581665,
79
+ "epoch": 0.37037037037037035,
80
+ "step": 5
81
+ },
82
+ {
83
+ "loss": 0.7796,
84
+ "grad_norm": 1.7338999509811401,
85
+ "learning_rate": 1.761904761904762e-05,
86
+ "rewards/chosen": 13.54969596862793,
87
+ "rewards/rejected": 0.8067520260810852,
88
+ "rewards/accuracies": 1.0,
89
+ "rewards/margins": 12.742944717407227,
90
+ "logps/chosen": -199.10501098632812,
91
+ "logps/rejected": -170.7447052001953,
92
+ "logits/chosen": -1.3969006538391113,
93
+ "logits/rejected": -1.5540287494659424,
94
+ "nll_loss": 0.7769691348075867,
95
+ "epoch": 0.4444444444444444,
96
+ "step": 6
97
+ },
98
+ {
99
+ "loss": 0.5148,
100
+ "grad_norm": 2.685260772705078,
101
+ "learning_rate": 1.7142857142857142e-05,
102
+ "rewards/chosen": 11.665142059326172,
103
+ "rewards/rejected": 5.737021446228027,
104
+ "rewards/accuracies": 1.0,
105
+ "rewards/margins": 5.928119659423828,
106
+ "logps/chosen": -161.44613647460938,
107
+ "logps/rejected": -160.96942138671875,
108
+ "logits/chosen": -1.4417845010757446,
109
+ "logits/rejected": -0.9561607241630554,
110
+ "nll_loss": 0.45274728536605835,
111
+ "epoch": 0.5185185185185185,
112
+ "step": 7
113
+ },
114
+ {
115
+ "loss": 0.4417,
116
+ "grad_norm": 3.1201682090759277,
117
+ "learning_rate": 1.6666666666666667e-05,
118
+ "rewards/chosen": 10.966083526611328,
119
+ "rewards/rejected": 4.790030002593994,
120
+ "rewards/accuracies": 1.0,
121
+ "rewards/margins": 6.176053047180176,
122
+ "logps/chosen": -137.63690185546875,
123
+ "logps/rejected": -56.03738021850586,
124
+ "logits/chosen": -0.8419337272644043,
125
+ "logits/rejected": -1.215024471282959,
126
+ "nll_loss": 0.39673712849617004,
127
+ "epoch": 0.5925925925925926,
128
+ "step": 8
129
+ },
130
+ {
131
+ "loss": 0.9551,
132
+ "grad_norm": 17.211788177490234,
133
+ "learning_rate": 1.6190476190476193e-05,
134
+ "rewards/chosen": 7.943333625793457,
135
+ "rewards/rejected": 7.321460247039795,
136
+ "rewards/accuracies": 0.5,
137
+ "rewards/margins": 0.6218737363815308,
138
+ "logps/chosen": -60.062042236328125,
139
+ "logps/rejected": -122.46002197265625,
140
+ "logits/chosen": -1.12626051902771,
141
+ "logits/rejected": -0.9878674745559692,
142
+ "nll_loss": 0.364607036113739,
143
+ "epoch": 0.6666666666666666,
144
+ "step": 9
145
+ },
146
+ {
147
+ "loss": 1.2544,
148
+ "grad_norm": 34.095741271972656,
149
+ "learning_rate": 1.5714285714285715e-05,
150
+ "rewards/chosen": 11.667631149291992,
151
+ "rewards/rejected": 6.291272163391113,
152
+ "rewards/accuracies": 0.5,
153
+ "rewards/margins": 5.376358985900879,
154
+ "logps/chosen": -108.86640930175781,
155
+ "logps/rejected": -281.20794677734375,
156
+ "logits/chosen": -1.3114888668060303,
157
+ "logits/rejected": -1.1465973854064941,
158
+ "nll_loss": 0.41387325525283813,
159
+ "epoch": 0.7407407407407407,
160
+ "step": 10
161
+ },
162
+ {
163
+ "loss": 0.3648,
164
+ "grad_norm": 2.947514057159424,
165
+ "learning_rate": 1.523809523809524e-05,
166
+ "rewards/chosen": 13.07333755493164,
167
+ "rewards/rejected": 3.849489212036133,
168
+ "rewards/accuracies": 1.0,
169
+ "rewards/margins": 9.223847389221191,
170
+ "logps/chosen": -97.10889434814453,
171
+ "logps/rejected": -63.29592514038086,
172
+ "logits/chosen": -1.9115753173828125,
173
+ "logits/rejected": -1.7226938009262085,
174
+ "nll_loss": 0.32489386200904846,
175
+ "epoch": 0.8148148148148148,
176
+ "step": 11
177
+ },
178
+ {
179
+ "loss": 0.3899,
180
+ "grad_norm": 3.3651766777038574,
181
+ "learning_rate": 1.4761904761904763e-05,
182
+ "rewards/chosen": 8.8883695602417,
183
+ "rewards/rejected": -0.298483282327652,
184
+ "rewards/accuracies": 1.0,
185
+ "rewards/margins": 9.186853408813477,
186
+ "logps/chosen": -51.296470642089844,
187
+ "logps/rejected": -242.6103515625,
188
+ "logits/chosen": -1.745258092880249,
189
+ "logits/rejected": -1.4840505123138428,
190
+ "nll_loss": 0.3581862449645996,
191
+ "epoch": 0.8888888888888888,
192
+ "step": 12
193
+ },
194
+ {
195
+ "loss": 0.4407,
196
+ "grad_norm": 3.5779449939727783,
197
+ "learning_rate": 1.4285714285714287e-05,
198
+ "rewards/chosen": 6.4146928787231445,
199
+ "rewards/rejected": 3.8673453330993652,
200
+ "rewards/accuracies": 1.0,
201
+ "rewards/margins": 2.5473475456237793,
202
+ "logps/chosen": -18.522918701171875,
203
+ "logps/rejected": -27.15282440185547,
204
+ "logits/chosen": -2.120323657989502,
205
+ "logits/rejected": -1.5977681875228882,
206
+ "nll_loss": 0.056797005236148834,
207
+ "epoch": 0.9629629629629629,
208
+ "step": 13
209
+ },
210
+ {
211
+ "loss": 0.3996,
212
+ "grad_norm": 1.5078325271606445,
213
+ "learning_rate": 1.3809523809523811e-05,
214
+ "rewards/chosen": 12.913629531860352,
215
+ "rewards/rejected": 0.0762176513671875,
216
+ "rewards/accuracies": 1.0,
217
+ "rewards/margins": 12.837411880493164,
218
+ "logps/chosen": -151.21395874023438,
219
+ "logps/rejected": -140.216796875,
220
+ "logits/chosen": -1.4412299394607544,
221
+ "logits/rejected": -1.572061538696289,
222
+ "nll_loss": 0.39793142676353455,
223
+ "epoch": 1.0,
224
+ "step": 14
225
+ },
226
+ {
227
+ "loss": 0.4154,
228
+ "grad_norm": 1.3567416667938232,
229
+ "learning_rate": 1.3333333333333333e-05,
230
+ "rewards/chosen": 11.772394180297852,
231
+ "rewards/rejected": 0.8464080691337585,
232
+ "rewards/accuracies": 1.0,
233
+ "rewards/margins": 10.925986289978027,
234
+ "logps/chosen": -149.495849609375,
235
+ "logps/rejected": -106.2137451171875,
236
+ "logits/chosen": -1.0617215633392334,
237
+ "logits/rejected": -1.2569652795791626,
238
+ "nll_loss": 0.4072629511356354,
239
+ "epoch": 1.074074074074074,
240
+ "step": 15
241
+ },
242
+ {
243
+ "loss": 0.468,
244
+ "grad_norm": 3.6080329418182373,
245
+ "learning_rate": 1.2857142857142859e-05,
246
+ "rewards/chosen": 6.804324150085449,
247
+ "rewards/rejected": 3.7284812927246094,
248
+ "rewards/accuracies": 1.0,
249
+ "rewards/margins": 3.07584285736084,
250
+ "logps/chosen": -17.333251953125,
251
+ "logps/rejected": -19.420581817626953,
252
+ "logits/chosen": -1.3385006189346313,
253
+ "logits/rejected": -1.2573401927947998,
254
+ "nll_loss": 0.11591336131095886,
255
+ "epoch": 1.1481481481481481,
256
+ "step": 16
257
+ },
258
+ {
259
+ "loss": 0.663,
260
+ "grad_norm": 5.882723808288574,
261
+ "learning_rate": 1.2380952380952383e-05,
262
+ "rewards/chosen": 12.403653144836426,
263
+ "rewards/rejected": 0.954088568687439,
264
+ "rewards/accuracies": 1.0,
265
+ "rewards/margins": 11.449564933776855,
266
+ "logps/chosen": -121.8620376586914,
267
+ "logps/rejected": -79.57422637939453,
268
+ "logits/chosen": -1.0053354501724243,
269
+ "logits/rejected": -1.1513221263885498,
270
+ "nll_loss": 0.561664342880249,
271
+ "epoch": 1.2222222222222223,
272
+ "step": 17
273
+ },
274
+ {
275
+ "loss": 0.3193,
276
+ "grad_norm": 1.6075787544250488,
277
+ "learning_rate": 1.1904761904761905e-05,
278
+ "rewards/chosen": 12.148561477661133,
279
+ "rewards/rejected": 4.71841287612915,
280
+ "rewards/accuracies": 1.0,
281
+ "rewards/margins": 7.430148124694824,
282
+ "logps/chosen": -86.73603057861328,
283
+ "logps/rejected": -72.05569458007812,
284
+ "logits/chosen": -1.3765130043029785,
285
+ "logits/rejected": -1.0658893585205078,
286
+ "nll_loss": 0.29516053199768066,
287
+ "epoch": 1.2962962962962963,
288
+ "step": 18
289
+ },
290
+ {
291
+ "loss": 0.0815,
292
+ "grad_norm": 1.4516865015029907,
293
+ "learning_rate": 1.1428571428571429e-05,
294
+ "rewards/chosen": 6.419312477111816,
295
+ "rewards/rejected": -1.014299750328064,
296
+ "rewards/accuracies": 1.0,
297
+ "rewards/margins": 7.433612823486328,
298
+ "logps/chosen": -18.83917999267578,
299
+ "logps/rejected": -221.62811279296875,
300
+ "logits/chosen": -2.136892080307007,
301
+ "logits/rejected": -1.0867035388946533,
302
+ "nll_loss": 0.05050197243690491,
303
+ "epoch": 1.3703703703703702,
304
+ "step": 19
305
+ },
306
+ {
307
+ "loss": 0.7592,
308
+ "grad_norm": 1.5342777967453003,
309
+ "learning_rate": 1.0952380952380955e-05,
310
+ "rewards/chosen": 15.713485717773438,
311
+ "rewards/rejected": 0.7484039664268494,
312
+ "rewards/accuracies": 1.0,
313
+ "rewards/margins": 14.965082168579102,
314
+ "logps/chosen": -229.06028747558594,
315
+ "logps/rejected": -223.26156616210938,
316
+ "logits/chosen": -1.3005579710006714,
317
+ "logits/rejected": -1.4877663850784302,
318
+ "nll_loss": 0.755147397518158,
319
+ "epoch": 1.4444444444444444,
320
+ "step": 20
321
+ },
322
+ {
323
+ "loss": 0.3657,
324
+ "grad_norm": 1.8852089643478394,
325
+ "learning_rate": 1.0476190476190477e-05,
326
+ "rewards/chosen": 6.322546482086182,
327
+ "rewards/rejected": -3.4525198936462402,
328
+ "rewards/accuracies": 1.0,
329
+ "rewards/margins": 9.775066375732422,
330
+ "logps/chosen": -83.4319076538086,
331
+ "logps/rejected": -150.83909606933594,
332
+ "logits/chosen": -1.4741220474243164,
333
+ "logits/rejected": -0.9544844627380371,
334
+ "nll_loss": 0.3566025495529175,
335
+ "epoch": 1.5185185185185186,
336
+ "step": 21
337
+ },
338
+ {
339
+ "loss": 0.3191,
340
+ "grad_norm": 1.680464267730713,
341
+ "learning_rate": 1e-05,
342
+ "rewards/chosen": 8.389806747436523,
343
+ "rewards/rejected": -1.2944848537445068,
344
+ "rewards/accuracies": 1.0,
345
+ "rewards/margins": 9.68429183959961,
346
+ "logps/chosen": -56.479522705078125,
347
+ "logps/rejected": -138.5536651611328,
348
+ "logits/chosen": -2.1388604640960693,
349
+ "logits/rejected": -1.9067153930664062,
350
+ "nll_loss": 0.3106623888015747,
351
+ "epoch": 1.5925925925925926,
352
+ "step": 22
353
+ },
354
+ {
355
+ "loss": 0.6475,
356
+ "grad_norm": 8.385740280151367,
357
+ "learning_rate": 9.523809523809525e-06,
358
+ "rewards/chosen": 12.101006507873535,
359
+ "rewards/rejected": 3.8977296352386475,
360
+ "rewards/accuracies": 1.0,
361
+ "rewards/margins": 8.203276634216309,
362
+ "logps/chosen": -89.65142822265625,
363
+ "logps/rejected": -209.59390258789062,
364
+ "logits/chosen": -1.6119561195373535,
365
+ "logits/rejected": -1.4711840152740479,
366
+ "nll_loss": 0.4897710382938385,
367
+ "epoch": 1.6666666666666665,
368
+ "step": 23
369
+ },
370
+ {
371
+ "loss": 0.4322,
372
+ "grad_norm": 11.931051254272461,
373
+ "learning_rate": 9.047619047619049e-06,
374
+ "rewards/chosen": 10.989448547363281,
375
+ "rewards/rejected": 1.7908951044082642,
376
+ "rewards/accuracies": 1.0,
377
+ "rewards/margins": 9.198553085327148,
378
+ "logps/chosen": -78.45639038085938,
379
+ "logps/rejected": -304.12548828125,
380
+ "logits/chosen": -1.8504806756973267,
381
+ "logits/rejected": -1.346926212310791,
382
+ "nll_loss": 0.2989135980606079,
383
+ "epoch": 1.7407407407407407,
384
+ "step": 24
385
+ },
386
+ {
387
+ "loss": 0.2474,
388
+ "grad_norm": 5.099134922027588,
389
+ "learning_rate": 8.571428571428571e-06,
390
+ "rewards/chosen": 8.742551803588867,
391
+ "rewards/rejected": 2.9189956188201904,
392
+ "rewards/accuracies": 1.0,
393
+ "rewards/margins": 5.823555946350098,
394
+ "logps/chosen": -27.888015747070312,
395
+ "logps/rejected": -167.5579833984375,
396
+ "logits/chosen": -1.988571286201477,
397
+ "logits/rejected": -1.6410770416259766,
398
+ "nll_loss": 0.11312854290008545,
399
+ "epoch": 1.8148148148148149,
400
+ "step": 25
401
+ },
402
+ {
403
+ "loss": 0.3587,
404
+ "grad_norm": 1.282354474067688,
405
+ "learning_rate": 8.095238095238097e-06,
406
+ "rewards/chosen": 16.977359771728516,
407
+ "rewards/rejected": 3.3890786170959473,
408
+ "rewards/accuracies": 1.0,
409
+ "rewards/margins": 13.58828067779541,
410
+ "logps/chosen": -117.91413879394531,
411
+ "logps/rejected": -40.2586555480957,
412
+ "logits/chosen": -1.5212771892547607,
413
+ "logits/rejected": -1.7528395652770996,
414
+ "nll_loss": 0.3565136790275574,
415
+ "epoch": 1.8888888888888888,
416
+ "step": 26
417
+ },
418
+ {
419
+ "loss": 0.4116,
420
+ "grad_norm": 1.4890203475952148,
421
+ "learning_rate": 7.61904761904762e-06,
422
+ "rewards/chosen": 19.562976837158203,
423
+ "rewards/rejected": 7.676817417144775,
424
+ "rewards/accuracies": 1.0,
425
+ "rewards/margins": 11.886159896850586,
426
+ "logps/chosen": -114.1052017211914,
427
+ "logps/rejected": -182.85549926757812,
428
+ "logits/chosen": -1.4092382192611694,
429
+ "logits/rejected": -1.2688117027282715,
430
+ "nll_loss": 0.40477409958839417,
431
+ "epoch": 1.9629629629629628,
432
+ "step": 27
433
+ },
434
+ {
435
+ "loss": 0.762,
436
+ "grad_norm": 2.4140851497650146,
437
+ "learning_rate": 7.1428571428571436e-06,
438
+ "rewards/chosen": 17.54005241394043,
439
+ "rewards/rejected": 2.19696044921875,
440
+ "rewards/accuracies": 1.0,
441
+ "rewards/margins": 15.34309196472168,
442
+ "logps/chosen": -215.520751953125,
443
+ "logps/rejected": -160.60769653320312,
444
+ "logits/chosen": -0.7868784070014954,
445
+ "logits/rejected": -1.2713123559951782,
446
+ "nll_loss": 0.7615573406219482,
447
+ "epoch": 2.0,
448
+ "step": 28
449
+ },
450
+ {
451
+ "loss": 0.4847,
452
+ "grad_norm": 1.2563648223876953,
453
+ "learning_rate": 6.666666666666667e-06,
454
+ "rewards/chosen": 16.273900985717773,
455
+ "rewards/rejected": 0.9482208490371704,
456
+ "rewards/accuracies": 1.0,
457
+ "rewards/margins": 15.32568073272705,
458
+ "logps/chosen": -119.34959411621094,
459
+ "logps/rejected": -149.0054473876953,
460
+ "logits/chosen": -1.9414124488830566,
461
+ "logits/rejected": -1.8630475997924805,
462
+ "nll_loss": 0.4807361960411072,
463
+ "epoch": 2.074074074074074,
464
+ "step": 29
465
+ },
466
+ {
467
+ "loss": 0.4099,
468
+ "grad_norm": 10.411107063293457,
469
+ "learning_rate": 6.1904761904761914e-06,
470
+ "rewards/chosen": 10.952390670776367,
471
+ "rewards/rejected": 1.5634475946426392,
472
+ "rewards/accuracies": 1.0,
473
+ "rewards/margins": 9.38894271850586,
474
+ "logps/chosen": -78.64167785644531,
475
+ "logps/rejected": -305.2627258300781,
476
+ "logits/chosen": -1.8730666637420654,
477
+ "logits/rejected": -1.4263302087783813,
478
+ "nll_loss": 0.30108290910720825,
479
+ "epoch": 2.148148148148148,
480
+ "step": 30
481
+ },
482
+ {
483
+ "loss": 0.3239,
484
+ "grad_norm": 1.3048899173736572,
485
+ "learning_rate": 5.7142857142857145e-06,
486
+ "rewards/chosen": 17.846054077148438,
487
+ "rewards/rejected": 5.169807434082031,
488
+ "rewards/accuracies": 1.0,
489
+ "rewards/margins": 12.676247596740723,
490
+ "logps/chosen": -99.78291320800781,
491
+ "logps/rejected": -274.7017517089844,
492
+ "logits/chosen": -1.7034916877746582,
493
+ "logits/rejected": -1.2672853469848633,
494
+ "nll_loss": 0.3191719353199005,
495
+ "epoch": 2.2222222222222223,
496
+ "step": 31
497
+ },
498
+ {
499
+ "loss": 0.2819,
500
+ "grad_norm": 1.5728684663772583,
501
+ "learning_rate": 5.2380952380952384e-06,
502
+ "rewards/chosen": 6.420802116394043,
503
+ "rewards/rejected": -3.7725648880004883,
504
+ "rewards/accuracies": 1.0,
505
+ "rewards/margins": 10.193367004394531,
506
+ "logps/chosen": -62.446197509765625,
507
+ "logps/rejected": -139.23294067382812,
508
+ "logits/chosen": -1.9392356872558594,
509
+ "logits/rejected": -1.721691608428955,
510
+ "nll_loss": 0.2750071883201599,
511
+ "epoch": 2.2962962962962963,
512
+ "step": 32
513
+ },
514
+ {
515
+ "loss": 0.1704,
516
+ "grad_norm": 1.0096286535263062,
517
+ "learning_rate": 4.761904761904762e-06,
518
+ "rewards/chosen": 12.679731369018555,
519
+ "rewards/rejected": 3.1536545753479004,
520
+ "rewards/accuracies": 1.0,
521
+ "rewards/margins": 9.526077270507812,
522
+ "logps/chosen": -36.873199462890625,
523
+ "logps/rejected": -75.43675994873047,
524
+ "logits/chosen": -1.7095279693603516,
525
+ "logits/rejected": -1.3212672472000122,
526
+ "nll_loss": 0.1618969440460205,
527
+ "epoch": 2.3703703703703702,
528
+ "step": 33
529
+ },
530
+ {
531
+ "loss": 0.2813,
532
+ "grad_norm": 7.387240409851074,
533
+ "learning_rate": 4.2857142857142855e-06,
534
+ "rewards/chosen": 9.613969802856445,
535
+ "rewards/rejected": 3.3647356033325195,
536
+ "rewards/accuracies": 1.0,
537
+ "rewards/margins": 6.249233722686768,
538
+ "logps/chosen": -37.82198715209961,
539
+ "logps/rejected": -277.5780944824219,
540
+ "logits/chosen": -1.7755906581878662,
541
+ "logits/rejected": -1.1926698684692383,
542
+ "nll_loss": 0.18008217215538025,
543
+ "epoch": 2.4444444444444446,
544
+ "step": 34
545
+ },
546
+ {
547
+ "loss": 0.5448,
548
+ "grad_norm": 1.4656599760055542,
549
+ "learning_rate": 3.80952380952381e-06,
550
+ "rewards/chosen": 17.712570190429688,
551
+ "rewards/rejected": 1.5176208019256592,
552
+ "rewards/accuracies": 1.0,
553
+ "rewards/margins": 16.194950103759766,
554
+ "logps/chosen": -170.93870544433594,
555
+ "logps/rejected": -148.507080078125,
556
+ "logits/chosen": -1.2818543910980225,
557
+ "logits/rejected": -1.6124653816223145,
558
+ "nll_loss": 0.5444608926773071,
559
+ "epoch": 2.5185185185185186,
560
+ "step": 35
561
+ },
562
+ {
563
+ "loss": 0.3955,
564
+ "grad_norm": 1.0936943292617798,
565
+ "learning_rate": 3.3333333333333333e-06,
566
+ "rewards/chosen": 15.414117813110352,
567
+ "rewards/rejected": 0.7488240003585815,
568
+ "rewards/accuracies": 1.0,
569
+ "rewards/margins": 14.665294647216797,
570
+ "logps/chosen": -134.19659423828125,
571
+ "logps/rejected": -53.766090393066406,
572
+ "logits/chosen": -1.4166557788848877,
573
+ "logits/rejected": -1.5128686428070068,
574
+ "nll_loss": 0.3940754532814026,
575
+ "epoch": 2.5925925925925926,
576
+ "step": 36
577
+ },
578
+ {
579
+ "loss": 0.5176,
580
+ "grad_norm": 3.5507962703704834,
581
+ "learning_rate": 2.8571428571428573e-06,
582
+ "rewards/chosen": 7.212784767150879,
583
+ "rewards/rejected": 0.3551967144012451,
584
+ "rewards/accuracies": 1.0,
585
+ "rewards/margins": 6.857588291168213,
586
+ "logps/chosen": -46.14129638671875,
587
+ "logps/rejected": -95.4029769897461,
588
+ "logits/chosen": -1.8642727136611938,
589
+ "logits/rejected": -1.5612514019012451,
590
+ "nll_loss": 0.26557254791259766,
591
+ "epoch": 2.6666666666666665,
592
+ "step": 37
593
+ },
594
+ {
595
+ "loss": 0.1928,
596
+ "grad_norm": 1.0856250524520874,
597
+ "learning_rate": 2.380952380952381e-06,
598
+ "rewards/chosen": 12.60528564453125,
599
+ "rewards/rejected": 2.5909197330474854,
600
+ "rewards/accuracies": 1.0,
601
+ "rewards/margins": 10.014365196228027,
602
+ "logps/chosen": -64.61308288574219,
603
+ "logps/rejected": -56.304283142089844,
604
+ "logits/chosen": -1.8486244678497314,
605
+ "logits/rejected": -1.6421642303466797,
606
+ "nll_loss": 0.1840396225452423,
607
+ "epoch": 2.7407407407407405,
608
+ "step": 38
609
+ },
610
+ {
611
+ "loss": 0.5181,
612
+ "grad_norm": 3.6901040077209473,
613
+ "learning_rate": 1.904761904761905e-06,
614
+ "rewards/chosen": 13.229158401489258,
615
+ "rewards/rejected": 6.178647994995117,
616
+ "rewards/accuracies": 1.0,
617
+ "rewards/margins": 7.050510883331299,
618
+ "logps/chosen": -147.8618621826172,
619
+ "logps/rejected": -170.39801025390625,
620
+ "logits/chosen": -1.745589017868042,
621
+ "logits/rejected": -1.6196589469909668,
622
+ "nll_loss": 0.4324379563331604,
623
+ "epoch": 2.814814814814815,
624
+ "step": 39
625
+ },
626
+ {
627
+ "loss": 0.539,
628
+ "grad_norm": 4.402906894683838,
629
+ "learning_rate": 1.4285714285714286e-06,
630
+ "rewards/chosen": 8.460529327392578,
631
+ "rewards/rejected": -1.933558702468872,
632
+ "rewards/accuracies": 1.0,
633
+ "rewards/margins": 10.394087791442871,
634
+ "logps/chosen": -66.4424057006836,
635
+ "logps/rejected": -183.2909393310547,
636
+ "logits/chosen": -1.5945682525634766,
637
+ "logits/rejected": -1.4925503730773926,
638
+ "nll_loss": 0.5039619207382202,
639
+ "epoch": 2.888888888888889,
640
+ "step": 40
641
+ },
642
+ {
643
+ "loss": 0.4705,
644
+ "grad_norm": 1.8300154209136963,
645
+ "learning_rate": 9.523809523809525e-07,
646
+ "rewards/chosen": 15.1546630859375,
647
+ "rewards/rejected": -0.15147249400615692,
648
+ "rewards/accuracies": 1.0,
649
+ "rewards/margins": 15.306135177612305,
650
+ "logps/chosen": -115.32080078125,
651
+ "logps/rejected": -54.289710998535156,
652
+ "logits/chosen": -1.0664377212524414,
653
+ "logits/rejected": -1.1847776174545288,
654
+ "nll_loss": 0.46403777599334717,
655
+ "epoch": 2.962962962962963,
656
+ "step": 41
657
+ },
658
+ {
659
+ "loss": 0.4014,
660
+ "grad_norm": 1.7773661613464355,
661
+ "learning_rate": 4.7619047619047623e-07,
662
+ "rewards/chosen": 12.449015617370605,
663
+ "rewards/rejected": 2.9710693359375,
664
+ "rewards/accuracies": 1.0,
665
+ "rewards/margins": 9.477946281433105,
666
+ "logps/chosen": -110.3365249633789,
667
+ "logps/rejected": -74.00520324707031,
668
+ "logits/chosen": -1.14561927318573,
669
+ "logits/rejected": -1.516902208328247,
670
+ "nll_loss": 0.3926566541194916,
671
+ "epoch": 3.0,
672
+ "step": 42
673
+ },
674
+ {
675
+ "train_runtime": 101.6987,
676
+ "train_samples_per_second": 0.796,
677
+ "train_steps_per_second": 0.413,
678
+ "total_flos": 0.0,
679
+ "train_loss": 0.49601077377086594,
680
+ "epoch": 3.0,
681
+ "step": 42
682
+ }
683
+ ]