Hugging Face's logo

42e
/
mathcompose-verifier-1.5b

Text Generation
PEFT
Safetensors
Transformers
lora
sft
trl
conversational
Model card Files
xet
 Community
42e commited on
Commit
0a40e3a
·
verified ·
1 Parent(s): 41ed4b4

Upload folder using huggingface_hub (part 2)

Browse files
Files changed (34) hide show
  1. checkpoint-550/optimizer.pt +1 -1
  2. checkpoint-550/trainer_state.json +280 -280
  3. checkpoint-600/adapter_config.json +5 -5
  4. checkpoint-600/adapter_model.safetensors +1 -1
  5. checkpoint-600/optimizer.pt +1 -1
  6. checkpoint-600/trainer_state.json +304 -304
  7. checkpoint-650/adapter_config.json +5 -5
  8. checkpoint-650/adapter_model.safetensors +1 -1
  9. checkpoint-650/optimizer.pt +1 -1
  10. checkpoint-650/trainer_state.json +329 -329
  11. checkpoint-700/adapter_config.json +5 -5
  12. checkpoint-700/adapter_model.safetensors +1 -1
  13. checkpoint-700/optimizer.pt +1 -1
  14. checkpoint-700/trainer_state.json +355 -355
  15. checkpoint-750/adapter_config.json +5 -5
  16. checkpoint-750/adapter_model.safetensors +1 -1
  17. checkpoint-750/optimizer.pt +1 -1
  18. checkpoint-750/trainer_state.json +379 -379
  19. checkpoint-800/adapter_config.json +5 -5
  20. checkpoint-800/adapter_model.safetensors +1 -1
  21. checkpoint-800/optimizer.pt +1 -1
  22. checkpoint-800/trainer_state.json +404 -404
  23. checkpoint-850/adapter_config.json +5 -5
  24. checkpoint-850/adapter_model.safetensors +1 -1
  25. checkpoint-850/optimizer.pt +1 -1
  26. checkpoint-850/trainer_state.json +428 -428
  27. checkpoint-900/adapter_config.json +5 -5
  28. checkpoint-900/adapter_model.safetensors +1 -1
  29. checkpoint-900/optimizer.pt +1 -1
  30. checkpoint-900/trainer_state.json +452 -452
  31. checkpoint-950/adapter_config.json +5 -5
  32. checkpoint-950/adapter_model.safetensors +1 -1
  33. checkpoint-950/optimizer.pt +1 -1
  34. checkpoint-950/trainer_state.json +477 -477
checkpoint-550/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8d89fda31c67881f3f8097f34f17aa9b52cff98fdf1d89131f03d0b1f147a85d
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bebdecf13a7286282e97584d3c5c553a3a4eb11f56d34884dfc59258b702f40a
3
  size 75481221
checkpoint-550/trainer_state.json CHANGED
@@ -10,674 +10,674 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  }
683
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  }
683
  ],
checkpoint-600/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-600/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:99b51e5573c493f818dc73cf3468f5fd8b40614ceec0b2595b62e02c1d65b105
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:104874f2e3207380404d94950687cdd49a0ef4119a84ec57cd9db0523ea443d3
3
  size 73911504
checkpoint-600/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a702e7a36a3e74d0e9239fd2dda090f740bb2415cafc716336d190d6cfcd1a38
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb3f152d250d50f8b725cd6cf35ef0f07f367507b83719df2d774039fabfc986
3
  size 75481221
checkpoint-600/trainer_state.json CHANGED
@@ -10,735 +10,735 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  }
744
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  }
744
  ],
checkpoint-650/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-650/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f759bf644828d9e51633b3ea127c358b1b23f79c77e7a04b941816692e4a1cbd
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06714d158eecbd3e5b1ee7217d99a9852ce2ddc16fc742ebc697716eacbfabf2
3
  size 73911504
checkpoint-650/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5a9dd46887bb831d66685198acdc49ddc06693cf7e3ceec9b405580c94116516
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:827ec557171a3f3033a52388ee46aae1a1c0253c211a11db72fd9d3569f25c85
3
  size 75481221
checkpoint-650/trainer_state.json CHANGED
@@ -10,796 +10,796 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  }
805
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  }
805
  ],
checkpoint-700/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-700/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0a8d0b3374f5e27bb367f0a86eaf1d5a0f17b9f9b4796b0f6872314b1bd58976
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c04d72c64ba9b8d98eab129bf5c29ebf1fad4d438ce79fc482622d4e426164f4
3
  size 73911504
checkpoint-700/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc99afc7a51ab43448e591a49c3fb5d1d84e621b6a9fb64abb95859b083e262a
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9aa5d7173e2a4737c2dc4d6b4faa504477596b88f339c4a90854a9c2e9e42988
3
  size 75481221
checkpoint-700/trainer_state.json CHANGED
@@ -10,857 +10,857 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  }
866
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  }
866
  ],
checkpoint-750/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-750/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:af0491ec58ad03820e66ac4abf78e7f17c7993a1f506e8279d8e4ce100317bcd
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:205b3271141e438a3829b2626144d2d442350b9ea67e610a6988de650a49446e
3
  size 73911504
checkpoint-750/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:330fd1771cc26450c97f464fa33bbb1849256512502a7affd103516d55094709
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b852701dbd464df886968a6826996c2dcd31eeaf8fad96d6cac91f12ab7cc0fb
3
  size 75481221
checkpoint-750/trainer_state.json CHANGED
@@ -10,918 +10,918 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  },
866
  {
867
- "entropy": 0.7161616936326027,
868
  "epoch": 1.3976377952755905,
869
- "grad_norm": 0.421875,
870
  "learning_rate": 0.00011608416337104065,
871
- "loss": 0.6723262786865234,
872
- "mean_token_accuracy": 0.8196182817220687,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
- "entropy": 0.6407245740294456,
878
  "epoch": 1.4173228346456692,
879
- "grad_norm": 0.443359375,
880
  "learning_rate": 0.00011398279233530829,
881
- "loss": 0.6370048522949219,
882
- "mean_token_accuracy": 0.8265632688999176,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
- "entropy": 0.704089181125164,
888
  "epoch": 1.4370078740157481,
889
- "grad_norm": 0.51953125,
890
  "learning_rate": 0.00011187510404420028,
891
- "loss": 0.645330286026001,
892
- "mean_token_accuracy": 0.8142861813306809,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
- "entropy": 0.7673430755734444,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
- "loss": 0.7298968315124512,
902
- "mean_token_accuracy": 0.8080587953329086,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
- "entropy": 0.6395664170384407,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
- "loss": 0.6174263477325439,
912
- "mean_token_accuracy": 0.8285290688276291,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
- "eval_entropy": 0.7358508281133793,
919
- "eval_loss": 0.8263605833053589,
920
- "eval_mean_token_accuracy": 0.7838875507866895,
921
  "eval_num_tokens": 8327961.0,
922
- "eval_runtime": 15.9403,
923
- "eval_samples_per_second": 26.787,
924
- "eval_steps_per_second": 3.388,
925
  "step": 750
926
  }
927
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  },
866
  {
867
+ "entropy": 0.7150849819183349,
868
  "epoch": 1.3976377952755905,
869
+ "grad_norm": 0.419921875,
870
  "learning_rate": 0.00011608416337104065,
871
+ "loss": 0.6717889785766602,
872
+ "mean_token_accuracy": 0.8194838553667069,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
+ "entropy": 0.6417959302663803,
878
  "epoch": 1.4173228346456692,
879
+ "grad_norm": 0.44140625,
880
  "learning_rate": 0.00011398279233530829,
881
+ "loss": 0.6370071411132813,
882
+ "mean_token_accuracy": 0.8262952893972397,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
+ "entropy": 0.7074607491493226,
888
  "epoch": 1.4370078740157481,
889
+ "grad_norm": 0.515625,
890
  "learning_rate": 0.00011187510404420028,
891
+ "loss": 0.6439510822296143,
892
+ "mean_token_accuracy": 0.8158842891454696,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
+ "entropy": 0.7658031359314919,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
+ "loss": 0.7316959857940674,
902
+ "mean_token_accuracy": 0.8061666846275329,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
+ "entropy": 0.6396550685167313,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
+ "loss": 0.6169118881225586,
912
+ "mean_token_accuracy": 0.8309739202260971,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
+ "eval_entropy": 0.7342568177867819,
919
+ "eval_loss": 0.8272306323051453,
920
+ "eval_mean_token_accuracy": 0.7827356656392416,
921
  "eval_num_tokens": 8327961.0,
922
+ "eval_runtime": 15.911,
923
+ "eval_samples_per_second": 26.837,
924
+ "eval_steps_per_second": 3.394,
925
  "step": 750
926
  }
927
  ],
checkpoint-800/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-800/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e417f2f8f7e9ff69840cd21bc8d2e393b4cf5fce2798938f71f7ddeff929a270
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a329d93b378e2a88048c3a2f808200bc09b39aefb5f1120b86625f55520b13e
3
  size 73911504
checkpoint-800/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b475899bc89c3feca2e82d3c121ce063724e654022958b5663ae66717874d5a2
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45677e45e0c1ba688f0c22849938e23dd88752216c7624d21dd5e6629256684f
3
  size 75481221
checkpoint-800/trainer_state.json CHANGED
@@ -10,979 +10,979 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  },
866
  {
867
- "entropy": 0.7161616936326027,
868
  "epoch": 1.3976377952755905,
869
- "grad_norm": 0.421875,
870
  "learning_rate": 0.00011608416337104065,
871
- "loss": 0.6723262786865234,
872
- "mean_token_accuracy": 0.8196182817220687,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
- "entropy": 0.6407245740294456,
878
  "epoch": 1.4173228346456692,
879
- "grad_norm": 0.443359375,
880
  "learning_rate": 0.00011398279233530829,
881
- "loss": 0.6370048522949219,
882
- "mean_token_accuracy": 0.8265632688999176,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
- "entropy": 0.704089181125164,
888
  "epoch": 1.4370078740157481,
889
- "grad_norm": 0.51953125,
890
  "learning_rate": 0.00011187510404420028,
891
- "loss": 0.645330286026001,
892
- "mean_token_accuracy": 0.8142861813306809,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
- "entropy": 0.7673430755734444,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
- "loss": 0.7298968315124512,
902
- "mean_token_accuracy": 0.8080587953329086,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
- "entropy": 0.6395664170384407,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
- "loss": 0.6174263477325439,
912
- "mean_token_accuracy": 0.8285290688276291,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
- "eval_entropy": 0.7358508281133793,
919
- "eval_loss": 0.8263605833053589,
920
- "eval_mean_token_accuracy": 0.7838875507866895,
921
  "eval_num_tokens": 8327961.0,
922
- "eval_runtime": 15.9403,
923
- "eval_samples_per_second": 26.787,
924
- "eval_steps_per_second": 3.388,
925
  "step": 750
926
  },
927
  {
928
- "entropy": 0.701068015396595,
929
  "epoch": 1.4960629921259843,
930
- "grad_norm": 0.65625,
931
  "learning_rate": 0.00010552366960573985,
932
- "loss": 0.6718067646026611,
933
- "mean_token_accuracy": 0.8166692137718201,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
- "entropy": 0.6750249415636063,
939
  "epoch": 1.515748031496063,
940
- "grad_norm": 0.41796875,
941
  "learning_rate": 0.00010340025665341684,
942
- "loss": 0.6471256256103516,
943
- "mean_token_accuracy": 0.8236999034881591,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
- "entropy": 0.6500637173652649,
949
  "epoch": 1.5354330708661417,
950
- "grad_norm": 0.470703125,
951
  "learning_rate": 0.00010127530750652642,
952
- "loss": 0.5989052295684815,
953
- "mean_token_accuracy": 0.8357917547225953,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
- "entropy": 0.6429416075348854,
959
  "epoch": 1.5551181102362204,
960
- "grad_norm": 0.69140625,
961
  "learning_rate": 9.914978219122791e-05,
962
- "loss": 0.6052910327911377,
963
- "mean_token_accuracy": 0.8295481771230697,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
- "entropy": 0.6374648019671441,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
- "loss": 0.6044469833374023,
973
- "mean_token_accuracy": 0.8315474331378937,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
- "eval_entropy": 0.7627564834223853,
980
- "eval_loss": 0.8170398473739624,
981
- "eval_mean_token_accuracy": 0.7843498874593664,
982
  "eval_num_tokens": 8886361.0,
983
- "eval_runtime": 15.9518,
984
- "eval_samples_per_second": 26.768,
985
- "eval_steps_per_second": 3.385,
986
  "step": 800
987
  }
988
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  },
866
  {
867
+ "entropy": 0.7150849819183349,
868
  "epoch": 1.3976377952755905,
869
+ "grad_norm": 0.419921875,
870
  "learning_rate": 0.00011608416337104065,
871
+ "loss": 0.6717889785766602,
872
+ "mean_token_accuracy": 0.8194838553667069,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
+ "entropy": 0.6417959302663803,
878
  "epoch": 1.4173228346456692,
879
+ "grad_norm": 0.44140625,
880
  "learning_rate": 0.00011398279233530829,
881
+ "loss": 0.6370071411132813,
882
+ "mean_token_accuracy": 0.8262952893972397,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
+ "entropy": 0.7074607491493226,
888
  "epoch": 1.4370078740157481,
889
+ "grad_norm": 0.515625,
890
  "learning_rate": 0.00011187510404420028,
891
+ "loss": 0.6439510822296143,
892
+ "mean_token_accuracy": 0.8158842891454696,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
+ "entropy": 0.7658031359314919,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
+ "loss": 0.7316959857940674,
902
+ "mean_token_accuracy": 0.8061666846275329,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
+ "entropy": 0.6396550685167313,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
+ "loss": 0.6169118881225586,
912
+ "mean_token_accuracy": 0.8309739202260971,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
+ "eval_entropy": 0.7342568177867819,
919
+ "eval_loss": 0.8272306323051453,
920
+ "eval_mean_token_accuracy": 0.7827356656392416,
921
  "eval_num_tokens": 8327961.0,
922
+ "eval_runtime": 15.911,
923
+ "eval_samples_per_second": 26.837,
924
+ "eval_steps_per_second": 3.394,
925
  "step": 750
926
  },
927
  {
928
+ "entropy": 0.6999662488698959,
929
  "epoch": 1.4960629921259843,
930
+ "grad_norm": 0.671875,
931
  "learning_rate": 0.00010552366960573985,
932
+ "loss": 0.6717596054077148,
933
+ "mean_token_accuracy": 0.8167526543140411,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
+ "entropy": 0.6736484602093696,
939
  "epoch": 1.515748031496063,
940
+ "grad_norm": 0.408203125,
941
  "learning_rate": 0.00010340025665341684,
942
+ "loss": 0.6454797744750976,
943
+ "mean_token_accuracy": 0.8241631388664246,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
+ "entropy": 0.6505535006523132,
949
  "epoch": 1.5354330708661417,
950
+ "grad_norm": 0.45703125,
951
  "learning_rate": 0.00010127530750652642,
952
+ "loss": 0.6001406669616699,
953
+ "mean_token_accuracy": 0.8359915494918824,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
+ "entropy": 0.643044576048851,
959
  "epoch": 1.5551181102362204,
960
+ "grad_norm": 0.6875,
961
  "learning_rate": 9.914978219122791e-05,
962
+ "loss": 0.6050069808959961,
963
+ "mean_token_accuracy": 0.8284690707921982,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
+ "entropy": 0.6357371553778648,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
+ "loss": 0.6026933193206787,
973
+ "mean_token_accuracy": 0.8313924312591553,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
+ "eval_entropy": 0.7611764465217237,
980
+ "eval_loss": 0.8170427083969116,
981
+ "eval_mean_token_accuracy": 0.7845815287695991,
982
  "eval_num_tokens": 8886361.0,
983
+ "eval_runtime": 15.9048,
984
+ "eval_samples_per_second": 26.847,
985
+ "eval_steps_per_second": 3.395,
986
  "step": 800
987
  }
988
  ],
checkpoint-850/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-850/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ac0098278dfc87eabece9004db97fc09b545d88e8125500724c8e9504353e828
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94d3171d77b31b8eea5b0ed03ba1f382e11bc2fc8f801f066a7abef925c89ad5
3
  size 73911504
checkpoint-850/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d84cce8cad39003f1502f0c1491914c415ea0b271245f59f346cf760df349ede
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb53b91af3b6e79548978b143889b832c4590ccab25582f60e359b7a677ae2b5
3
  size 75481221
checkpoint-850/trainer_state.json CHANGED
@@ -10,1040 +10,1040 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  },
866
  {
867
- "entropy": 0.7161616936326027,
868
  "epoch": 1.3976377952755905,
869
- "grad_norm": 0.421875,
870
  "learning_rate": 0.00011608416337104065,
871
- "loss": 0.6723262786865234,
872
- "mean_token_accuracy": 0.8196182817220687,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
- "entropy": 0.6407245740294456,
878
  "epoch": 1.4173228346456692,
879
- "grad_norm": 0.443359375,
880
  "learning_rate": 0.00011398279233530829,
881
- "loss": 0.6370048522949219,
882
- "mean_token_accuracy": 0.8265632688999176,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
- "entropy": 0.704089181125164,
888
  "epoch": 1.4370078740157481,
889
- "grad_norm": 0.51953125,
890
  "learning_rate": 0.00011187510404420028,
891
- "loss": 0.645330286026001,
892
- "mean_token_accuracy": 0.8142861813306809,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
- "entropy": 0.7673430755734444,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
- "loss": 0.7298968315124512,
902
- "mean_token_accuracy": 0.8080587953329086,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
- "entropy": 0.6395664170384407,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
- "loss": 0.6174263477325439,
912
- "mean_token_accuracy": 0.8285290688276291,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
- "eval_entropy": 0.7358508281133793,
919
- "eval_loss": 0.8263605833053589,
920
- "eval_mean_token_accuracy": 0.7838875507866895,
921
  "eval_num_tokens": 8327961.0,
922
- "eval_runtime": 15.9403,
923
- "eval_samples_per_second": 26.787,
924
- "eval_steps_per_second": 3.388,
925
  "step": 750
926
  },
927
  {
928
- "entropy": 0.701068015396595,
929
  "epoch": 1.4960629921259843,
930
- "grad_norm": 0.65625,
931
  "learning_rate": 0.00010552366960573985,
932
- "loss": 0.6718067646026611,
933
- "mean_token_accuracy": 0.8166692137718201,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
- "entropy": 0.6750249415636063,
939
  "epoch": 1.515748031496063,
940
- "grad_norm": 0.41796875,
941
  "learning_rate": 0.00010340025665341684,
942
- "loss": 0.6471256256103516,
943
- "mean_token_accuracy": 0.8236999034881591,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
- "entropy": 0.6500637173652649,
949
  "epoch": 1.5354330708661417,
950
- "grad_norm": 0.470703125,
951
  "learning_rate": 0.00010127530750652642,
952
- "loss": 0.5989052295684815,
953
- "mean_token_accuracy": 0.8357917547225953,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
- "entropy": 0.6429416075348854,
959
  "epoch": 1.5551181102362204,
960
- "grad_norm": 0.69140625,
961
  "learning_rate": 9.914978219122791e-05,
962
- "loss": 0.6052910327911377,
963
- "mean_token_accuracy": 0.8295481771230697,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
- "entropy": 0.6374648019671441,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
- "loss": 0.6044469833374023,
973
- "mean_token_accuracy": 0.8315474331378937,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
- "eval_entropy": 0.7627564834223853,
980
- "eval_loss": 0.8170398473739624,
981
- "eval_mean_token_accuracy": 0.7843498874593664,
982
  "eval_num_tokens": 8886361.0,
983
- "eval_runtime": 15.9518,
984
- "eval_samples_per_second": 26.768,
985
- "eval_steps_per_second": 3.385,
986
  "step": 800
987
  },
988
  {
989
- "entropy": 0.6787230268120765,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
- "loss": 0.6268706321716309,
994
- "mean_token_accuracy": 0.8275486946105957,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
- "entropy": 0.7012800723314285,
1000
  "epoch": 1.6141732283464567,
1001
- "grad_norm": 0.5390625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
- "loss": 0.652205228805542,
1004
- "mean_token_accuracy": 0.8216399610042572,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
- "entropy": 0.6063105046749115,
1010
  "epoch": 1.6338582677165354,
1011
- "grad_norm": 0.8125,
1012
  "learning_rate": 9.066111976982712e-05,
1013
- "loss": 0.597124719619751,
1014
- "mean_token_accuracy": 0.836309888958931,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
- "entropy": 0.6953654140233994,
1020
  "epoch": 1.6535433070866141,
1021
- "grad_norm": 0.5,
1022
  "learning_rate": 8.854710793399947e-05,
1023
- "loss": 0.6501974105834961,
1024
- "mean_token_accuracy": 0.8178625762462616,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
- "entropy": 0.6784902378916741,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
- "loss": 0.636347484588623,
1034
- "mean_token_accuracy": 0.8207234263420105,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
- "eval_entropy": 0.7619620506410245,
1041
- "eval_loss": 0.8146092295646667,
1042
- "eval_mean_token_accuracy": 0.7850696554890385,
1043
  "eval_num_tokens": 9429496.0,
1044
- "eval_runtime": 15.937,
1045
- "eval_samples_per_second": 26.793,
1046
- "eval_steps_per_second": 3.388,
1047
  "step": 850
1048
  }
1049
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  },
866
  {
867
+ "entropy": 0.7150849819183349,
868
  "epoch": 1.3976377952755905,
869
+ "grad_norm": 0.419921875,
870
  "learning_rate": 0.00011608416337104065,
871
+ "loss": 0.6717889785766602,
872
+ "mean_token_accuracy": 0.8194838553667069,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
+ "entropy": 0.6417959302663803,
878
  "epoch": 1.4173228346456692,
879
+ "grad_norm": 0.44140625,
880
  "learning_rate": 0.00011398279233530829,
881
+ "loss": 0.6370071411132813,
882
+ "mean_token_accuracy": 0.8262952893972397,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
+ "entropy": 0.7074607491493226,
888
  "epoch": 1.4370078740157481,
889
+ "grad_norm": 0.515625,
890
  "learning_rate": 0.00011187510404420028,
891
+ "loss": 0.6439510822296143,
892
+ "mean_token_accuracy": 0.8158842891454696,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
+ "entropy": 0.7658031359314919,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
+ "loss": 0.7316959857940674,
902
+ "mean_token_accuracy": 0.8061666846275329,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
+ "entropy": 0.6396550685167313,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
+ "loss": 0.6169118881225586,
912
+ "mean_token_accuracy": 0.8309739202260971,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
+ "eval_entropy": 0.7342568177867819,
919
+ "eval_loss": 0.8272306323051453,
920
+ "eval_mean_token_accuracy": 0.7827356656392416,
921
  "eval_num_tokens": 8327961.0,
922
+ "eval_runtime": 15.911,
923
+ "eval_samples_per_second": 26.837,
924
+ "eval_steps_per_second": 3.394,
925
  "step": 750
926
  },
927
  {
928
+ "entropy": 0.6999662488698959,
929
  "epoch": 1.4960629921259843,
930
+ "grad_norm": 0.671875,
931
  "learning_rate": 0.00010552366960573985,
932
+ "loss": 0.6717596054077148,
933
+ "mean_token_accuracy": 0.8167526543140411,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
+ "entropy": 0.6736484602093696,
939
  "epoch": 1.515748031496063,
940
+ "grad_norm": 0.408203125,
941
  "learning_rate": 0.00010340025665341684,
942
+ "loss": 0.6454797744750976,
943
+ "mean_token_accuracy": 0.8241631388664246,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
+ "entropy": 0.6505535006523132,
949
  "epoch": 1.5354330708661417,
950
+ "grad_norm": 0.45703125,
951
  "learning_rate": 0.00010127530750652642,
952
+ "loss": 0.6001406669616699,
953
+ "mean_token_accuracy": 0.8359915494918824,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
+ "entropy": 0.643044576048851,
959
  "epoch": 1.5551181102362204,
960
+ "grad_norm": 0.6875,
961
  "learning_rate": 9.914978219122791e-05,
962
+ "loss": 0.6050069808959961,
963
+ "mean_token_accuracy": 0.8284690707921982,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
+ "entropy": 0.6357371553778648,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
+ "loss": 0.6026933193206787,
973
+ "mean_token_accuracy": 0.8313924312591553,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
+ "eval_entropy": 0.7611764465217237,
980
+ "eval_loss": 0.8170427083969116,
981
+ "eval_mean_token_accuracy": 0.7845815287695991,
982
  "eval_num_tokens": 8886361.0,
983
+ "eval_runtime": 15.9048,
984
+ "eval_samples_per_second": 26.847,
985
+ "eval_steps_per_second": 3.395,
986
  "step": 800
987
  },
988
  {
989
+ "entropy": 0.6775295034050941,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
+ "loss": 0.6273195743560791,
994
+ "mean_token_accuracy": 0.8267990559339523,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
+ "entropy": 0.7035147309303283,
1000
  "epoch": 1.6141732283464567,
1001
+ "grad_norm": 0.53515625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
+ "loss": 0.6532655715942383,
1004
+ "mean_token_accuracy": 0.8209419757127762,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
+ "entropy": 0.6083378851413727,
1010
  "epoch": 1.6338582677165354,
1011
+ "grad_norm": 0.796875,
1012
  "learning_rate": 9.066111976982712e-05,
1013
+ "loss": 0.5965664386749268,
1014
+ "mean_token_accuracy": 0.8344648003578186,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
+ "entropy": 0.6942630469799042,
1020
  "epoch": 1.6535433070866141,
1021
+ "grad_norm": 0.50390625,
1022
  "learning_rate": 8.854710793399947e-05,
1023
+ "loss": 0.6506452560424805,
1024
+ "mean_token_accuracy": 0.8177429378032685,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
+ "entropy": 0.6778671249747277,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
+ "loss": 0.6361007213592529,
1034
+ "mean_token_accuracy": 0.8213603168725967,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
+ "eval_entropy": 0.7621777179064574,
1041
+ "eval_loss": 0.8149666786193848,
1042
+ "eval_mean_token_accuracy": 0.7850644290447235,
1043
  "eval_num_tokens": 9429496.0,
1044
+ "eval_runtime": 15.9323,
1045
+ "eval_samples_per_second": 26.801,
1046
+ "eval_steps_per_second": 3.389,
1047
  "step": 850
1048
  }
1049
  ],
checkpoint-900/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-900/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f0e697e200ddde36f0a5457357338468c0e6028c32f8eba7a7b84213593e6ec7
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:081475b671ef25b7480ea96876a56cf62f2f84e8f52c80d5757101a20e65c9f5
3
  size 73911504
checkpoint-900/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a339286e18b921850ffc8f7d9f1cea99c2c8fceaff7aa300a841a2aa45f49f90
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8364d39a08b4f202219dfcb432bd9f353d304ccab2e43fefc4d9453e39ea04d7
3
  size 75481221
checkpoint-900/trainer_state.json CHANGED
@@ -10,1101 +10,1101 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  },
866
  {
867
- "entropy": 0.7161616936326027,
868
  "epoch": 1.3976377952755905,
869
- "grad_norm": 0.421875,
870
  "learning_rate": 0.00011608416337104065,
871
- "loss": 0.6723262786865234,
872
- "mean_token_accuracy": 0.8196182817220687,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
- "entropy": 0.6407245740294456,
878
  "epoch": 1.4173228346456692,
879
- "grad_norm": 0.443359375,
880
  "learning_rate": 0.00011398279233530829,
881
- "loss": 0.6370048522949219,
882
- "mean_token_accuracy": 0.8265632688999176,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
- "entropy": 0.704089181125164,
888
  "epoch": 1.4370078740157481,
889
- "grad_norm": 0.51953125,
890
  "learning_rate": 0.00011187510404420028,
891
- "loss": 0.645330286026001,
892
- "mean_token_accuracy": 0.8142861813306809,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
- "entropy": 0.7673430755734444,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
- "loss": 0.7298968315124512,
902
- "mean_token_accuracy": 0.8080587953329086,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
- "entropy": 0.6395664170384407,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
- "loss": 0.6174263477325439,
912
- "mean_token_accuracy": 0.8285290688276291,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
- "eval_entropy": 0.7358508281133793,
919
- "eval_loss": 0.8263605833053589,
920
- "eval_mean_token_accuracy": 0.7838875507866895,
921
  "eval_num_tokens": 8327961.0,
922
- "eval_runtime": 15.9403,
923
- "eval_samples_per_second": 26.787,
924
- "eval_steps_per_second": 3.388,
925
  "step": 750
926
  },
927
  {
928
- "entropy": 0.701068015396595,
929
  "epoch": 1.4960629921259843,
930
- "grad_norm": 0.65625,
931
  "learning_rate": 0.00010552366960573985,
932
- "loss": 0.6718067646026611,
933
- "mean_token_accuracy": 0.8166692137718201,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
- "entropy": 0.6750249415636063,
939
  "epoch": 1.515748031496063,
940
- "grad_norm": 0.41796875,
941
  "learning_rate": 0.00010340025665341684,
942
- "loss": 0.6471256256103516,
943
- "mean_token_accuracy": 0.8236999034881591,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
- "entropy": 0.6500637173652649,
949
  "epoch": 1.5354330708661417,
950
- "grad_norm": 0.470703125,
951
  "learning_rate": 0.00010127530750652642,
952
- "loss": 0.5989052295684815,
953
- "mean_token_accuracy": 0.8357917547225953,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
- "entropy": 0.6429416075348854,
959
  "epoch": 1.5551181102362204,
960
- "grad_norm": 0.69140625,
961
  "learning_rate": 9.914978219122791e-05,
962
- "loss": 0.6052910327911377,
963
- "mean_token_accuracy": 0.8295481771230697,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
- "entropy": 0.6374648019671441,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
- "loss": 0.6044469833374023,
973
- "mean_token_accuracy": 0.8315474331378937,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
- "eval_entropy": 0.7627564834223853,
980
- "eval_loss": 0.8170398473739624,
981
- "eval_mean_token_accuracy": 0.7843498874593664,
982
  "eval_num_tokens": 8886361.0,
983
- "eval_runtime": 15.9518,
984
- "eval_samples_per_second": 26.768,
985
- "eval_steps_per_second": 3.385,
986
  "step": 800
987
  },
988
  {
989
- "entropy": 0.6787230268120765,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
- "loss": 0.6268706321716309,
994
- "mean_token_accuracy": 0.8275486946105957,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
- "entropy": 0.7012800723314285,
1000
  "epoch": 1.6141732283464567,
1001
- "grad_norm": 0.5390625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
- "loss": 0.652205228805542,
1004
- "mean_token_accuracy": 0.8216399610042572,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
- "entropy": 0.6063105046749115,
1010
  "epoch": 1.6338582677165354,
1011
- "grad_norm": 0.8125,
1012
  "learning_rate": 9.066111976982712e-05,
1013
- "loss": 0.597124719619751,
1014
- "mean_token_accuracy": 0.836309888958931,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
- "entropy": 0.6953654140233994,
1020
  "epoch": 1.6535433070866141,
1021
- "grad_norm": 0.5,
1022
  "learning_rate": 8.854710793399947e-05,
1023
- "loss": 0.6501974105834961,
1024
- "mean_token_accuracy": 0.8178625762462616,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
- "entropy": 0.6784902378916741,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
- "loss": 0.636347484588623,
1034
- "mean_token_accuracy": 0.8207234263420105,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
- "eval_entropy": 0.7619620506410245,
1041
- "eval_loss": 0.8146092295646667,
1042
- "eval_mean_token_accuracy": 0.7850696554890385,
1043
  "eval_num_tokens": 9429496.0,
1044
- "eval_runtime": 15.937,
1045
- "eval_samples_per_second": 26.793,
1046
- "eval_steps_per_second": 3.388,
1047
  "step": 850
1048
  },
1049
  {
1050
- "entropy": 0.6777935117483139,
1051
  "epoch": 1.6929133858267718,
1052
- "grad_norm": 0.478515625,
1053
  "learning_rate": 8.433555984116937e-05,
1054
- "loss": 0.6113362312316895,
1055
- "mean_token_accuracy": 0.8263391226530075,
1056
  "num_tokens": 9544270.0,
1057
  "step": 860
1058
  },
1059
  {
1060
- "entropy": 0.6014138415455819,
1061
  "epoch": 1.7125984251968505,
1062
  "grad_norm": 0.53125,
1063
  "learning_rate": 8.22399263103314e-05,
1064
- "loss": 0.6068051338195801,
1065
- "mean_token_accuracy": 0.8325248181819915,
1066
  "num_tokens": 9654198.0,
1067
  "step": 870
1068
  },
1069
  {
1070
- "entropy": 0.6414325833320618,
1071
  "epoch": 1.7322834645669292,
1072
- "grad_norm": 0.69140625,
1073
  "learning_rate": 8.015231656460712e-05,
1074
- "loss": 0.6324831008911133,
1075
- "mean_token_accuracy": 0.834089708328247,
1076
  "num_tokens": 9769457.0,
1077
  "step": 880
1078
  },
1079
  {
1080
- "entropy": 0.6839869737625122,
1081
  "epoch": 1.7519685039370079,
1082
  "grad_norm": 0.5390625,
1083
  "learning_rate": 7.807367376066889e-05,
1084
- "loss": 0.6296841144561768,
1085
- "mean_token_accuracy": 0.8278504878282547,
1086
  "num_tokens": 9875831.0,
1087
  "step": 890
1088
  },
1089
  {
1090
- "entropy": 0.6922678858041763,
1091
  "epoch": 1.7716535433070866,
1092
- "grad_norm": 0.48046875,
1093
  "learning_rate": 7.60049370040338e-05,
1094
- "loss": 0.6325759887695312,
1095
- "mean_token_accuracy": 0.8210694164037704,
1096
  "num_tokens": 9991758.0,
1097
  "step": 900
1098
  },
1099
  {
1100
  "epoch": 1.7716535433070866,
1101
- "eval_entropy": 0.7464290934580343,
1102
- "eval_loss": 0.8118129372596741,
1103
- "eval_mean_token_accuracy": 0.7863925894101461,
1104
  "eval_num_tokens": 9991758.0,
1105
- "eval_runtime": 15.9375,
1106
- "eval_samples_per_second": 26.792,
1107
- "eval_steps_per_second": 3.388,
1108
  "step": 900
1109
  }
1110
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  },
866
  {
867
+ "entropy": 0.7150849819183349,
868
  "epoch": 1.3976377952755905,
869
+ "grad_norm": 0.419921875,
870
  "learning_rate": 0.00011608416337104065,
871
+ "loss": 0.6717889785766602,
872
+ "mean_token_accuracy": 0.8194838553667069,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
+ "entropy": 0.6417959302663803,
878
  "epoch": 1.4173228346456692,
879
+ "grad_norm": 0.44140625,
880
  "learning_rate": 0.00011398279233530829,
881
+ "loss": 0.6370071411132813,
882
+ "mean_token_accuracy": 0.8262952893972397,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
+ "entropy": 0.7074607491493226,
888
  "epoch": 1.4370078740157481,
889
+ "grad_norm": 0.515625,
890
  "learning_rate": 0.00011187510404420028,
891
+ "loss": 0.6439510822296143,
892
+ "mean_token_accuracy": 0.8158842891454696,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
+ "entropy": 0.7658031359314919,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
+ "loss": 0.7316959857940674,
902
+ "mean_token_accuracy": 0.8061666846275329,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
+ "entropy": 0.6396550685167313,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
+ "loss": 0.6169118881225586,
912
+ "mean_token_accuracy": 0.8309739202260971,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
+ "eval_entropy": 0.7342568177867819,
919
+ "eval_loss": 0.8272306323051453,
920
+ "eval_mean_token_accuracy": 0.7827356656392416,
921
  "eval_num_tokens": 8327961.0,
922
+ "eval_runtime": 15.911,
923
+ "eval_samples_per_second": 26.837,
924
+ "eval_steps_per_second": 3.394,
925
  "step": 750
926
  },
927
  {
928
+ "entropy": 0.6999662488698959,
929
  "epoch": 1.4960629921259843,
930
+ "grad_norm": 0.671875,
931
  "learning_rate": 0.00010552366960573985,
932
+ "loss": 0.6717596054077148,
933
+ "mean_token_accuracy": 0.8167526543140411,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
+ "entropy": 0.6736484602093696,
939
  "epoch": 1.515748031496063,
940
+ "grad_norm": 0.408203125,
941
  "learning_rate": 0.00010340025665341684,
942
+ "loss": 0.6454797744750976,
943
+ "mean_token_accuracy": 0.8241631388664246,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
+ "entropy": 0.6505535006523132,
949
  "epoch": 1.5354330708661417,
950
+ "grad_norm": 0.45703125,
951
  "learning_rate": 0.00010127530750652642,
952
+ "loss": 0.6001406669616699,
953
+ "mean_token_accuracy": 0.8359915494918824,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
+ "entropy": 0.643044576048851,
959
  "epoch": 1.5551181102362204,
960
+ "grad_norm": 0.6875,
961
  "learning_rate": 9.914978219122791e-05,
962
+ "loss": 0.6050069808959961,
963
+ "mean_token_accuracy": 0.8284690707921982,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
+ "entropy": 0.6357371553778648,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
+ "loss": 0.6026933193206787,
973
+ "mean_token_accuracy": 0.8313924312591553,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
+ "eval_entropy": 0.7611764465217237,
980
+ "eval_loss": 0.8170427083969116,
981
+ "eval_mean_token_accuracy": 0.7845815287695991,
982
  "eval_num_tokens": 8886361.0,
983
+ "eval_runtime": 15.9048,
984
+ "eval_samples_per_second": 26.847,
985
+ "eval_steps_per_second": 3.395,
986
  "step": 800
987
  },
988
  {
989
+ "entropy": 0.6775295034050941,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
+ "loss": 0.6273195743560791,
994
+ "mean_token_accuracy": 0.8267990559339523,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
+ "entropy": 0.7035147309303283,
1000
  "epoch": 1.6141732283464567,
1001
+ "grad_norm": 0.53515625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
+ "loss": 0.6532655715942383,
1004
+ "mean_token_accuracy": 0.8209419757127762,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
+ "entropy": 0.6083378851413727,
1010
  "epoch": 1.6338582677165354,
1011
+ "grad_norm": 0.796875,
1012
  "learning_rate": 9.066111976982712e-05,
1013
+ "loss": 0.5965664386749268,
1014
+ "mean_token_accuracy": 0.8344648003578186,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
+ "entropy": 0.6942630469799042,
1020
  "epoch": 1.6535433070866141,
1021
+ "grad_norm": 0.50390625,
1022
  "learning_rate": 8.854710793399947e-05,
1023
+ "loss": 0.6506452560424805,
1024
+ "mean_token_accuracy": 0.8177429378032685,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
+ "entropy": 0.6778671249747277,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
+ "loss": 0.6361007213592529,
1034
+ "mean_token_accuracy": 0.8213603168725967,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
+ "eval_entropy": 0.7621777179064574,
1041
+ "eval_loss": 0.8149666786193848,
1042
+ "eval_mean_token_accuracy": 0.7850644290447235,
1043
  "eval_num_tokens": 9429496.0,
1044
+ "eval_runtime": 15.9323,
1045
+ "eval_samples_per_second": 26.801,
1046
+ "eval_steps_per_second": 3.389,
1047
  "step": 850
1048
  },
1049
  {
1050
+ "entropy": 0.6798793867230415,
1051
  "epoch": 1.6929133858267718,
1052
+ "grad_norm": 0.46875,
1053
  "learning_rate": 8.433555984116937e-05,
1054
+ "loss": 0.613386869430542,
1055
+ "mean_token_accuracy": 0.8275126874446869,
1056
  "num_tokens": 9544270.0,
1057
  "step": 860
1058
  },
1059
  {
1060
+ "entropy": 0.6023602589964867,
1061
  "epoch": 1.7125984251968505,
1062
  "grad_norm": 0.53125,
1063
  "learning_rate": 8.22399263103314e-05,
1064
+ "loss": 0.6046566963195801,
1065
+ "mean_token_accuracy": 0.832113066315651,
1066
  "num_tokens": 9654198.0,
1067
  "step": 870
1068
  },
1069
  {
1070
+ "entropy": 0.640959857404232,
1071
  "epoch": 1.7322834645669292,
1072
+ "grad_norm": 0.6875,
1073
  "learning_rate": 8.015231656460712e-05,
1074
+ "loss": 0.6335668563842773,
1075
+ "mean_token_accuracy": 0.8338181763887406,
1076
  "num_tokens": 9769457.0,
1077
  "step": 880
1078
  },
1079
  {
1080
+ "entropy": 0.6831659495830535,
1081
  "epoch": 1.7519685039370079,
1082
  "grad_norm": 0.5390625,
1083
  "learning_rate": 7.807367376066889e-05,
1084
+ "loss": 0.6274240970611572,
1085
+ "mean_token_accuracy": 0.8270208179950714,
1086
  "num_tokens": 9875831.0,
1087
  "step": 890
1088
  },
1089
  {
1090
+ "entropy": 0.691785454750061,
1091
  "epoch": 1.7716535433070866,
1092
+ "grad_norm": 0.48828125,
1093
  "learning_rate": 7.60049370040338e-05,
1094
+ "loss": 0.631044578552246,
1095
+ "mean_token_accuracy": 0.8213751494884491,
1096
  "num_tokens": 9991758.0,
1097
  "step": 900
1098
  },
1099
  {
1100
  "epoch": 1.7716535433070866,
1101
+ "eval_entropy": 0.741917089731605,
1102
+ "eval_loss": 0.8126867413520813,
1103
+ "eval_mean_token_accuracy": 0.7860870074342798,
1104
  "eval_num_tokens": 9991758.0,
1105
+ "eval_runtime": 15.9328,
1106
+ "eval_samples_per_second": 26.8,
1107
+ "eval_steps_per_second": 3.389,
1108
  "step": 900
1109
  }
1110
  ],
checkpoint-950/adapter_config.json CHANGED
@@ -30,13 +30,13 @@
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
 
33
  "down_proj",
34
- "v_proj",
35
- "o_proj",
36
- "q_proj",
37
  "up_proj",
38
- "k_proj",
39
- "gate_proj"
 
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
 
30
  "rank_pattern": {},
31
  "revision": null,
32
  "target_modules": [
33
+ "k_proj",
34
  "down_proj",
35
+ "gate_proj",
 
 
36
  "up_proj",
37
+ "o_proj",
38
+ "v_proj",
39
+ "q_proj"
40
  ],
41
  "target_parameters": null,
42
  "task_type": "CAUSAL_LM",
checkpoint-950/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e365c2b05217002f0b37c0646859949eb6bc40e7833707dff976ec3055187612
3
  size 73911504
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:814e9cfdd7f64bc96092360cb8ed0d5f58a3c834828b87b6c9024aeac9ca62e0
3
  size 73911504
checkpoint-950/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1a654a4c5de101e6b39cb96ef0802dae984ffb277d53573127dbca4957e51dea
3
  size 75481221
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1dfc0428c30b8e066767e5b21b334e7d972b5608846788855d9ce61bd4027f1
3
  size 75481221
checkpoint-950/trainer_state.json CHANGED
@@ -10,1162 +10,1162 @@
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "entropy": 1.4378354728221894,
14
  "epoch": 0.01968503937007874,
15
- "grad_norm": 0.62109375,
16
  "learning_rate": 3.91304347826087e-05,
17
- "loss": 1.657682228088379,
18
- "mean_token_accuracy": 0.6598833322525024,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
- "entropy": 1.5141301095485686,
24
  "epoch": 0.03937007874015748,
25
- "grad_norm": 0.44140625,
26
  "learning_rate": 8.260869565217392e-05,
27
- "loss": 1.4646138191223144,
28
- "mean_token_accuracy": 0.6834615051746369,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
- "entropy": 1.2295305937528611,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
- "loss": 1.1134957313537597,
38
- "mean_token_accuracy": 0.7453642427921295,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
- "entropy": 1.0573274463415145,
44
  "epoch": 0.07874015748031496,
45
- "grad_norm": 0.474609375,
46
  "learning_rate": 0.00016956521739130436,
47
- "loss": 1.077296829223633,
48
- "mean_token_accuracy": 0.7446383863687516,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
- "entropy": 0.952904561161995,
54
  "epoch": 0.0984251968503937,
55
- "grad_norm": 0.50390625,
56
  "learning_rate": 0.00019999796688511753,
57
- "loss": 0.9590332984924317,
58
- "mean_token_accuracy": 0.768164387345314,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
- "eval_entropy": 1.0900797645250957,
65
- "eval_loss": 1.0484484434127808,
66
- "eval_mean_token_accuracy": 0.7461776600943671,
67
  "eval_num_tokens": 551175.0,
68
- "eval_runtime": 15.9866,
69
- "eval_samples_per_second": 26.71,
70
- "eval_steps_per_second": 3.378,
71
  "step": 50
72
  },
73
  {
74
- "entropy": 0.9487997680902481,
75
  "epoch": 0.11811023622047244,
76
- "grad_norm": 0.59375,
77
  "learning_rate": 0.0001999618249203163,
78
- "loss": 0.9577752113342285,
79
- "mean_token_accuracy": 0.7689193069934845,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
- "entropy": 1.0506175458431244,
85
  "epoch": 0.1377952755905512,
86
- "grad_norm": 0.5390625,
87
  "learning_rate": 0.00019988052141977201,
88
- "loss": 0.9976933479309082,
89
- "mean_token_accuracy": 0.756487911939621,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
- "entropy": 0.8528705358505249,
95
  "epoch": 0.15748031496062992,
96
- "grad_norm": 0.5078125,
97
  "learning_rate": 0.0001997540931154166,
98
- "loss": 0.8407608032226562,
99
- "mean_token_accuracy": 0.7868376106023789,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
- "entropy": 0.9220433413982392,
105
  "epoch": 0.17716535433070865,
106
- "grad_norm": 0.484375,
107
  "learning_rate": 0.00019958259712601903,
108
- "loss": 0.8958203315734863,
109
- "mean_token_accuracy": 0.7736223608255386,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
- "entropy": 0.8399994730949402,
115
  "epoch": 0.1968503937007874,
116
- "grad_norm": 0.51953125,
117
  "learning_rate": 0.00019936611093137983,
118
- "loss": 0.8394058227539063,
119
- "mean_token_accuracy": 0.7876519203186035,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
- "eval_entropy": 0.9908565746413337,
126
- "eval_loss": 0.949418842792511,
127
- "eval_mean_token_accuracy": 0.7609190278583102,
128
  "eval_num_tokens": 1103999.0,
129
- "eval_runtime": 15.9475,
130
- "eval_samples_per_second": 26.775,
131
- "eval_steps_per_second": 3.386,
132
  "step": 100
133
  },
134
  {
135
- "entropy": 0.8772778362035751,
136
  "epoch": 0.21653543307086615,
137
- "grad_norm": 0.40625,
138
  "learning_rate": 0.00019910473233732668,
139
- "loss": 0.8347702026367188,
140
- "mean_token_accuracy": 0.7877911299467086,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
- "entropy": 0.8745568007230758,
146
  "epoch": 0.23622047244094488,
147
- "grad_norm": 0.609375,
148
  "learning_rate": 0.00019879857943152674,
149
- "loss": 0.8359953880310058,
150
- "mean_token_accuracy": 0.7790884494781494,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
- "entropy": 0.878146781027317,
156
  "epoch": 0.2559055118110236,
157
- "grad_norm": 0.5390625,
158
  "learning_rate": 0.00019844779053013637,
159
- "loss": 0.8335110664367675,
160
- "mean_token_accuracy": 0.7791220843791962,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
- "entropy": 0.788033875823021,
166
  "epoch": 0.2755905511811024,
167
- "grad_norm": 0.546875,
168
  "learning_rate": 0.00019805252411531126,
169
- "loss": 0.7967269420623779,
170
- "mean_token_accuracy": 0.7931749552488327,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
- "entropy": 0.8667207360267639,
176
  "epoch": 0.2952755905511811,
177
- "grad_norm": 0.5546875,
178
  "learning_rate": 0.0001976129587636064,
179
- "loss": 0.8101954460144043,
180
- "mean_token_accuracy": 0.7908060193061829,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
- "eval_entropy": 0.9391163366812246,
187
- "eval_loss": 0.9146928191184998,
188
- "eval_mean_token_accuracy": 0.7670937886944523,
189
  "eval_num_tokens": 1654462.0,
190
- "eval_runtime": 15.956,
191
- "eval_samples_per_second": 26.761,
192
- "eval_steps_per_second": 3.384,
193
  "step": 150
194
  },
195
  {
196
- "entropy": 0.769407157599926,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
- "loss": 0.7655523777008056,
201
- "mean_token_accuracy": 0.8053037762641907,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
- "entropy": 0.8818381905555726,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
- "loss": 0.8408863067626953,
211
- "mean_token_accuracy": 0.7815578579902649,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
- "entropy": 0.8344995141029358,
217
  "epoch": 0.3543307086614173,
218
- "grad_norm": 0.4765625,
219
  "learning_rate": 0.0001960305545112445,
220
- "loss": 0.8205550193786622,
221
- "mean_token_accuracy": 0.7871369570493698,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
- "entropy": 0.8058241099119187,
227
  "epoch": 0.37401574803149606,
228
- "grad_norm": 0.494140625,
229
  "learning_rate": 0.0001954159780522056,
230
- "loss": 0.8002825736999511,
231
- "mean_token_accuracy": 0.7913093209266663,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
- "entropy": 0.8938606142997741,
237
  "epoch": 0.3937007874015748,
238
- "grad_norm": 0.61328125,
239
  "learning_rate": 0.00019475829381570566,
240
- "loss": 0.8305434226989746,
241
- "mean_token_accuracy": 0.7836446255445481,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
- "eval_entropy": 0.9502577009024443,
248
- "eval_loss": 0.8978050351142883,
249
- "eval_mean_token_accuracy": 0.7696934717672842,
250
  "eval_num_tokens": 2195494.0,
251
- "eval_runtime": 15.9419,
252
- "eval_samples_per_second": 26.785,
253
- "eval_steps_per_second": 3.387,
254
  "step": 200
255
  },
256
  {
257
- "entropy": 0.7890492826700211,
258
  "epoch": 0.41338582677165353,
259
- "grad_norm": 0.40625,
260
  "learning_rate": 0.0001940577989354773,
261
- "loss": 0.7953387260437011,
262
- "mean_token_accuracy": 0.7982505321502685,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
- "entropy": 0.8576261162757873,
268
  "epoch": 0.4330708661417323,
269
- "grad_norm": 0.66015625,
270
  "learning_rate": 0.00019331480988658083,
271
- "loss": 0.8120486259460449,
272
- "mean_token_accuracy": 0.7829198718070984,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
- "entropy": 0.8514308303594589,
278
  "epoch": 0.452755905511811,
279
- "grad_norm": 0.470703125,
280
  "learning_rate": 0.00019252966234242468,
281
- "loss": 0.8203784942626953,
282
- "mean_token_accuracy": 0.7819343030452728,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
- "entropy": 0.8994888842105866,
288
  "epoch": 0.47244094488188976,
289
- "grad_norm": 0.59765625,
290
  "learning_rate": 0.00019170271102311238,
291
- "loss": 0.9019557952880859,
292
- "mean_token_accuracy": 0.7712395042181015,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
- "entropy": 0.7910601049661636,
298
  "epoch": 0.4921259842519685,
299
- "grad_norm": 0.494140625,
300
  "learning_rate": 0.00019083432953518418,
301
- "loss": 0.8107678413391113,
302
- "mean_token_accuracy": 0.7996705889701843,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
- "eval_entropy": 0.9037868281205496,
309
- "eval_loss": 0.8843846917152405,
310
- "eval_mean_token_accuracy": 0.7717091352851303,
311
  "eval_num_tokens": 2741031.0,
312
- "eval_runtime": 15.93,
313
- "eval_samples_per_second": 26.805,
314
- "eval_steps_per_second": 3.39,
315
  "step": 250
316
  },
317
  {
318
- "entropy": 0.7740790158510208,
319
  "epoch": 0.5118110236220472,
320
- "grad_norm": 0.57421875,
321
  "learning_rate": 0.00018992491020282618,
322
- "loss": 0.7579652309417725,
323
- "mean_token_accuracy": 0.7955388814210892,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
- "entropy": 0.8572690159082412,
329
  "epoch": 0.531496062992126,
330
- "grad_norm": 1.171875,
331
  "learning_rate": 0.0001889748638906231,
332
- "loss": 0.8072081565856933,
333
- "mean_token_accuracy": 0.7910655379295349,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
- "entropy": 0.8885004252195359,
339
  "epoch": 0.5511811023622047,
340
- "grad_norm": 0.451171875,
341
  "learning_rate": 0.00018798461981793462,
342
- "loss": 0.8460770606994629,
343
- "mean_token_accuracy": 0.7776093751192092,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
- "entropy": 0.829613895714283,
349
  "epoch": 0.5708661417322834,
350
- "grad_norm": 0.4453125,
351
  "learning_rate": 0.00018695462536497934,
352
- "loss": 0.823792839050293,
353
- "mean_token_accuracy": 0.7848509013652801,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
- "entropy": 0.8341330245137215,
359
  "epoch": 0.5905511811023622,
360
- "grad_norm": 0.490234375,
361
  "learning_rate": 0.00018588534587071385,
362
- "loss": 0.810818099975586,
363
- "mean_token_accuracy": 0.7906568497419357,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
- "eval_entropy": 0.8445847459413387,
370
- "eval_loss": 0.8718247413635254,
371
- "eval_mean_token_accuracy": 0.774838043583764,
372
  "eval_num_tokens": 3297319.0,
373
- "eval_runtime": 15.9561,
374
- "eval_samples_per_second": 26.761,
375
- "eval_steps_per_second": 3.384,
376
  "step": 300
377
  },
378
  {
379
- "entropy": 0.8317201763391495,
380
  "epoch": 0.610236220472441,
381
- "grad_norm": 0.396484375,
382
  "learning_rate": 0.00018477726442259806,
383
- "loss": 0.80075101852417,
384
- "mean_token_accuracy": 0.7886052161455155,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
- "entropy": 0.8597566410899162,
390
  "epoch": 0.6299212598425197,
391
- "grad_norm": 0.404296875,
392
  "learning_rate": 0.00018363088163834252,
393
- "loss": 0.8123328208923339,
394
- "mean_token_accuracy": 0.7854531407356262,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
- "entropy": 0.8519140958786011,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
- "loss": 0.8214571952819825,
404
- "mean_token_accuracy": 0.7828344345092774,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
- "entropy": 0.7781533747911453,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
- "loss": 0.753899097442627,
414
- "mean_token_accuracy": 0.8011120349168778,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
- "entropy": 0.8384901583194733,
420
  "epoch": 0.6889763779527559,
421
- "grad_norm": 0.51171875,
422
  "learning_rate": 0.0001799671895953471,
423
- "loss": 0.8112489700317382,
424
- "mean_token_accuracy": 0.7898134678602219,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
- "eval_entropy": 0.8795759567507991,
431
- "eval_loss": 0.8568941354751587,
432
- "eval_mean_token_accuracy": 0.7754804182935644,
433
  "eval_num_tokens": 3870377.0,
434
- "eval_runtime": 15.9337,
435
- "eval_samples_per_second": 26.798,
436
- "eval_steps_per_second": 3.389,
437
  "step": 350
438
  },
439
  {
440
- "entropy": 0.8008699864149094,
441
  "epoch": 0.7086614173228346,
442
- "grad_norm": 0.498046875,
443
  "learning_rate": 0.00017867295016916167,
444
- "loss": 0.7585730075836181,
445
- "mean_token_accuracy": 0.7942175149917603,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
- "entropy": 0.9035080641508102,
451
  "epoch": 0.7283464566929134,
452
- "grad_norm": 0.48046875,
453
  "learning_rate": 0.0001773431672617132,
454
- "loss": 0.8653995513916015,
455
- "mean_token_accuracy": 0.7730168104171753,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
- "entropy": 0.760843014717102,
461
  "epoch": 0.7480314960629921,
462
- "grad_norm": 0.49609375,
463
  "learning_rate": 0.00017597844165273286,
464
- "loss": 0.7377382278442383,
465
- "mean_token_accuracy": 0.7972156703472137,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
- "entropy": 0.8075762838125229,
471
  "epoch": 0.7677165354330708,
472
- "grad_norm": 0.447265625,
473
  "learning_rate": 0.00017457938990863908,
474
- "loss": 0.7615089893341065,
475
- "mean_token_accuracy": 0.8005614727735519,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
- "entropy": 0.8942124575376511,
481
  "epoch": 0.7874015748031497,
482
- "grad_norm": 0.48828125,
483
  "learning_rate": 0.00017314664410398033,
484
- "loss": 0.8365015983581543,
485
- "mean_token_accuracy": 0.7790710270404816,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
- "eval_entropy": 0.8613758815659417,
492
- "eval_loss": 0.8455358743667603,
493
- "eval_mean_token_accuracy": 0.7789811304321995,
494
  "eval_num_tokens": 4414002.0,
495
- "eval_runtime": 16.2401,
496
- "eval_samples_per_second": 26.293,
497
- "eval_steps_per_second": 3.325,
498
  "step": 400
499
  },
500
  {
501
- "entropy": 0.7841676041483879,
502
  "epoch": 0.8070866141732284,
503
- "grad_norm": 0.57421875,
504
  "learning_rate": 0.0001716808515358714,
505
- "loss": 0.8027224540710449,
506
- "mean_token_accuracy": 0.79732064306736,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
- "entropy": 0.7710117936134339,
512
  "epoch": 0.8267716535433071,
513
- "grad_norm": 0.4453125,
514
  "learning_rate": 0.00017018267443155268,
515
- "loss": 0.7185665130615234,
516
- "mean_token_accuracy": 0.807221668958664,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
- "entropy": 0.8494665056467057,
522
  "epoch": 0.8464566929133859,
523
- "grad_norm": 0.453125,
524
  "learning_rate": 0.00016865278964920378,
525
- "loss": 0.8273701667785645,
526
- "mean_token_accuracy": 0.7870708376169204,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
- "entropy": 0.7938957527279854,
532
  "epoch": 0.8661417322834646,
533
- "grad_norm": 0.345703125,
534
  "learning_rate": 0.00016709188837214732,
535
- "loss": 0.7776453971862793,
536
- "mean_token_accuracy": 0.7911081612110138,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
- "entropy": 0.7948901996016502,
542
  "epoch": 0.8858267716535433,
543
- "grad_norm": 0.376953125,
544
  "learning_rate": 0.0001655006757965806,
545
- "loss": 0.7554664611816406,
546
- "mean_token_accuracy": 0.7939331024885178,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
- "eval_entropy": 0.8405309479545664,
553
- "eval_loss": 0.8371886610984802,
554
- "eval_mean_token_accuracy": 0.7799682672376986,
555
  "eval_num_tokens": 4984410.0,
556
- "eval_runtime": 15.9419,
557
- "eval_samples_per_second": 26.785,
558
- "eval_steps_per_second": 3.387,
559
  "step": 450
560
  },
561
  {
562
- "entropy": 0.8764406844973565,
563
  "epoch": 0.905511811023622,
564
- "grad_norm": 0.5625,
565
  "learning_rate": 0.0001638798708129768,
566
- "loss": 0.8325652122497559,
567
- "mean_token_accuracy": 0.7821455717086792,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
- "entropy": 0.8582975655794144,
573
  "epoch": 0.9251968503937008,
574
- "grad_norm": 0.3828125,
575
  "learning_rate": 0.00016223020568129854,
576
- "loss": 0.8309861183166504,
577
- "mean_token_accuracy": 0.7791573971509933,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
- "entropy": 0.8273941457271576,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
- "loss": 0.7878972053527832,
587
- "mean_token_accuracy": 0.7900911033153534,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
- "entropy": 0.8146103203296662,
593
  "epoch": 0.9645669291338582,
594
- "grad_norm": 0.42578125,
595
  "learning_rate": 0.00015884728887017162,
596
- "loss": 0.7865399837493896,
597
- "mean_token_accuracy": 0.7947310745716095,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
- "entropy": 0.6613125756382943,
603
  "epoch": 0.984251968503937,
604
- "grad_norm": 0.41796875,
605
  "learning_rate": 0.0001571155655513606,
606
- "loss": 0.6725417137145996,
607
- "mean_token_accuracy": 0.816859421133995,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
- "eval_entropy": 0.8118620448642306,
614
- "eval_loss": 0.8308539986610413,
615
- "eval_mean_token_accuracy": 0.781522473803273,
616
  "eval_num_tokens": 5544086.0,
617
- "eval_runtime": 15.9383,
618
- "eval_samples_per_second": 26.791,
619
- "eval_steps_per_second": 3.388,
620
  "step": 500
621
  },
622
  {
623
- "entropy": 0.7347942754626274,
624
  "epoch": 1.0039370078740157,
625
- "grad_norm": 0.39453125,
626
  "learning_rate": 0.0001553580381152559,
627
- "loss": 0.6915054798126221,
628
- "mean_token_accuracy": 0.8176700383424759,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
- "entropy": 0.7550317242741584,
634
  "epoch": 1.0236220472440944,
635
- "grad_norm": 0.4921875,
636
  "learning_rate": 0.0001535755005913602,
637
- "loss": 0.6865068912506104,
638
- "mean_token_accuracy": 0.8140248268842697,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
- "entropy": 0.7479684263467788,
644
  "epoch": 1.0433070866141732,
645
- "grad_norm": 0.60546875,
646
  "learning_rate": 0.00015176875830842938,
647
- "loss": 0.7037695407867431,
648
- "mean_token_accuracy": 0.8133333414793015,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
- "entropy": 0.6792907789349556,
654
  "epoch": 1.0629921259842519,
655
- "grad_norm": 0.439453125,
656
  "learning_rate": 0.0001499386275306348,
657
- "loss": 0.640275239944458,
658
- "mean_token_accuracy": 0.8249545454978943,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
- "entropy": 0.6819084867835045,
664
  "epoch": 1.0826771653543308,
665
- "grad_norm": 0.41015625,
666
  "learning_rate": 0.0001480859350887852,
667
- "loss": 0.6469451904296875,
668
- "mean_token_accuracy": 0.8235166579484939,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
- "eval_entropy": 0.8042021377219094,
675
- "eval_loss": 0.8352769017219543,
676
- "eval_mean_token_accuracy": 0.7806271998970596,
677
  "eval_num_tokens": 6112085.0,
678
- "eval_runtime": 15.9298,
679
- "eval_samples_per_second": 26.805,
680
- "eval_steps_per_second": 3.39,
681
  "step": 550
682
  },
683
  {
684
- "entropy": 0.7026952266693115,
685
  "epoch": 1.1023622047244095,
686
- "grad_norm": 0.47265625,
687
  "learning_rate": 0.00014621151800677434,
688
- "loss": 0.667764139175415,
689
- "mean_token_accuracy": 0.8164290696382522,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
- "entropy": 0.6083555489778518,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
- "loss": 0.5703843116760254,
699
- "mean_token_accuracy": 0.8396952480077744,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
- "entropy": 0.6823922500014306,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
- "loss": 0.6488718032836914,
709
- "mean_token_accuracy": 0.8208951652050018,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
- "entropy": 0.6331134095788002,
715
  "epoch": 1.1614173228346456,
716
- "grad_norm": 0.482421875,
717
  "learning_rate": 0.00014046643408282305,
718
- "loss": 0.5916334629058838,
719
- "mean_token_accuracy": 0.8340157747268677,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
- "entropy": 0.6203767985105515,
725
  "epoch": 1.1811023622047245,
726
- "grad_norm": 0.45703125,
727
  "learning_rate": 0.00013851367921340017,
728
- "loss": 0.5833216667175293,
729
- "mean_token_accuracy": 0.8345949351787567,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
- "eval_entropy": 0.7756788493306549,
736
- "eval_loss": 0.8351659178733826,
737
- "eval_mean_token_accuracy": 0.7810865651678156,
738
  "eval_num_tokens": 6676922.0,
739
- "eval_runtime": 15.9481,
740
- "eval_samples_per_second": 26.774,
741
- "eval_steps_per_second": 3.386,
742
  "step": 600
743
  },
744
  {
745
- "entropy": 0.645186324417591,
746
  "epoch": 1.2007874015748032,
747
- "grad_norm": 0.498046875,
748
  "learning_rate": 0.0001365435243325051,
749
- "loss": 0.6237170219421386,
750
- "mean_token_accuracy": 0.8277532249689102,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
- "entropy": 0.6759833052754403,
756
  "epoch": 1.220472440944882,
757
- "grad_norm": 0.546875,
758
  "learning_rate": 0.00013455685953213194,
759
- "loss": 0.6462036609649658,
760
- "mean_token_accuracy": 0.8247299671173096,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
- "entropy": 0.6737162366509437,
766
  "epoch": 1.2401574803149606,
767
- "grad_norm": 0.48828125,
768
  "learning_rate": 0.0001325545823632554,
769
- "loss": 0.6107028484344482,
770
- "mean_token_accuracy": 0.8260864973068237,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
- "entropy": 0.6702337503433228,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
- "loss": 0.6504123210906982,
780
- "mean_token_accuracy": 0.818900191783905,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
- "entropy": 0.7739008516073227,
786
  "epoch": 1.279527559055118,
787
- "grad_norm": 0.56640625,
788
  "learning_rate": 0.00012850681598259195,
789
- "loss": 0.7173595905303956,
790
- "mean_token_accuracy": 0.7997365325689316,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
- "eval_entropy": 0.7500080190323017,
797
- "eval_loss": 0.8297256231307983,
798
- "eval_mean_token_accuracy": 0.7822851461392862,
799
  "eval_num_tokens": 7218739.0,
800
- "eval_runtime": 15.9417,
801
- "eval_samples_per_second": 26.785,
802
- "eval_steps_per_second": 3.387,
803
  "step": 650
804
  },
805
  {
806
- "entropy": 0.7080179423093795,
807
  "epoch": 1.2992125984251968,
808
- "grad_norm": 0.53125,
809
  "learning_rate": 0.0001264631555023856,
810
- "loss": 0.661363172531128,
811
- "mean_token_accuracy": 0.8172380715608597,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
- "entropy": 0.6184108316898346,
817
  "epoch": 1.3188976377952755,
818
- "grad_norm": 0.78515625,
819
  "learning_rate": 0.00012440753929063826,
820
- "loss": 0.6013145923614502,
821
- "mean_token_accuracy": 0.8273262619972229,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
- "entropy": 0.71297377794981,
827
  "epoch": 1.3385826771653544,
828
- "grad_norm": 0.5078125,
829
  "learning_rate": 0.00012234089604973303,
830
- "loss": 0.6702734470367432,
831
- "mean_token_accuracy": 0.8151450455188751,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
- "entropy": 0.6389088705182076,
837
  "epoch": 1.358267716535433,
838
- "grad_norm": 0.50390625,
839
  "learning_rate": 0.0001202641594639305,
840
- "loss": 0.6133567810058593,
841
- "mean_token_accuracy": 0.8298202604055405,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
- "entropy": 0.7520714432001114,
847
  "epoch": 1.3779527559055118,
848
- "grad_norm": 0.5390625,
849
  "learning_rate": 0.00011817826777754171,
850
- "loss": 0.6847642421722412,
851
- "mean_token_accuracy": 0.8055585384368896,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
- "eval_entropy": 0.7791200037355777,
858
- "eval_loss": 0.8222253322601318,
859
- "eval_mean_token_accuracy": 0.7843283326537521,
860
  "eval_num_tokens": 7782831.0,
861
- "eval_runtime": 15.9494,
862
- "eval_samples_per_second": 26.772,
863
- "eval_steps_per_second": 3.386,
864
  "step": 700
865
  },
866
  {
867
- "entropy": 0.7161616936326027,
868
  "epoch": 1.3976377952755905,
869
- "grad_norm": 0.421875,
870
  "learning_rate": 0.00011608416337104065,
871
- "loss": 0.6723262786865234,
872
- "mean_token_accuracy": 0.8196182817220687,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
- "entropy": 0.6407245740294456,
878
  "epoch": 1.4173228346456692,
879
- "grad_norm": 0.443359375,
880
  "learning_rate": 0.00011398279233530829,
881
- "loss": 0.6370048522949219,
882
- "mean_token_accuracy": 0.8265632688999176,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
- "entropy": 0.704089181125164,
888
  "epoch": 1.4370078740157481,
889
- "grad_norm": 0.51953125,
890
  "learning_rate": 0.00011187510404420028,
891
- "loss": 0.645330286026001,
892
- "mean_token_accuracy": 0.8142861813306809,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
- "entropy": 0.7673430755734444,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
- "loss": 0.7298968315124512,
902
- "mean_token_accuracy": 0.8080587953329086,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
- "entropy": 0.6395664170384407,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
- "loss": 0.6174263477325439,
912
- "mean_token_accuracy": 0.8285290688276291,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
- "eval_entropy": 0.7358508281133793,
919
- "eval_loss": 0.8263605833053589,
920
- "eval_mean_token_accuracy": 0.7838875507866895,
921
  "eval_num_tokens": 8327961.0,
922
- "eval_runtime": 15.9403,
923
- "eval_samples_per_second": 26.787,
924
- "eval_steps_per_second": 3.388,
925
  "step": 750
926
  },
927
  {
928
- "entropy": 0.701068015396595,
929
  "epoch": 1.4960629921259843,
930
- "grad_norm": 0.65625,
931
  "learning_rate": 0.00010552366960573985,
932
- "loss": 0.6718067646026611,
933
- "mean_token_accuracy": 0.8166692137718201,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
- "entropy": 0.6750249415636063,
939
  "epoch": 1.515748031496063,
940
- "grad_norm": 0.41796875,
941
  "learning_rate": 0.00010340025665341684,
942
- "loss": 0.6471256256103516,
943
- "mean_token_accuracy": 0.8236999034881591,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
- "entropy": 0.6500637173652649,
949
  "epoch": 1.5354330708661417,
950
- "grad_norm": 0.470703125,
951
  "learning_rate": 0.00010127530750652642,
952
- "loss": 0.5989052295684815,
953
- "mean_token_accuracy": 0.8357917547225953,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
- "entropy": 0.6429416075348854,
959
  "epoch": 1.5551181102362204,
960
- "grad_norm": 0.69140625,
961
  "learning_rate": 9.914978219122791e-05,
962
- "loss": 0.6052910327911377,
963
- "mean_token_accuracy": 0.8295481771230697,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
- "entropy": 0.6374648019671441,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
- "loss": 0.6044469833374023,
973
- "mean_token_accuracy": 0.8315474331378937,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
- "eval_entropy": 0.7627564834223853,
980
- "eval_loss": 0.8170398473739624,
981
- "eval_mean_token_accuracy": 0.7843498874593664,
982
  "eval_num_tokens": 8886361.0,
983
- "eval_runtime": 15.9518,
984
- "eval_samples_per_second": 26.768,
985
- "eval_steps_per_second": 3.385,
986
  "step": 800
987
  },
988
  {
989
- "entropy": 0.6787230268120765,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
- "loss": 0.6268706321716309,
994
- "mean_token_accuracy": 0.8275486946105957,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
- "entropy": 0.7012800723314285,
1000
  "epoch": 1.6141732283464567,
1001
- "grad_norm": 0.5390625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
- "loss": 0.652205228805542,
1004
- "mean_token_accuracy": 0.8216399610042572,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
- "entropy": 0.6063105046749115,
1010
  "epoch": 1.6338582677165354,
1011
- "grad_norm": 0.8125,
1012
  "learning_rate": 9.066111976982712e-05,
1013
- "loss": 0.597124719619751,
1014
- "mean_token_accuracy": 0.836309888958931,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
- "entropy": 0.6953654140233994,
1020
  "epoch": 1.6535433070866141,
1021
- "grad_norm": 0.5,
1022
  "learning_rate": 8.854710793399947e-05,
1023
- "loss": 0.6501974105834961,
1024
- "mean_token_accuracy": 0.8178625762462616,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
- "entropy": 0.6784902378916741,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
- "loss": 0.636347484588623,
1034
- "mean_token_accuracy": 0.8207234263420105,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
- "eval_entropy": 0.7619620506410245,
1041
- "eval_loss": 0.8146092295646667,
1042
- "eval_mean_token_accuracy": 0.7850696554890385,
1043
  "eval_num_tokens": 9429496.0,
1044
- "eval_runtime": 15.937,
1045
- "eval_samples_per_second": 26.793,
1046
- "eval_steps_per_second": 3.388,
1047
  "step": 850
1048
  },
1049
  {
1050
- "entropy": 0.6777935117483139,
1051
  "epoch": 1.6929133858267718,
1052
- "grad_norm": 0.478515625,
1053
  "learning_rate": 8.433555984116937e-05,
1054
- "loss": 0.6113362312316895,
1055
- "mean_token_accuracy": 0.8263391226530075,
1056
  "num_tokens": 9544270.0,
1057
  "step": 860
1058
  },
1059
  {
1060
- "entropy": 0.6014138415455819,
1061
  "epoch": 1.7125984251968505,
1062
  "grad_norm": 0.53125,
1063
  "learning_rate": 8.22399263103314e-05,
1064
- "loss": 0.6068051338195801,
1065
- "mean_token_accuracy": 0.8325248181819915,
1066
  "num_tokens": 9654198.0,
1067
  "step": 870
1068
  },
1069
  {
1070
- "entropy": 0.6414325833320618,
1071
  "epoch": 1.7322834645669292,
1072
- "grad_norm": 0.69140625,
1073
  "learning_rate": 8.015231656460712e-05,
1074
- "loss": 0.6324831008911133,
1075
- "mean_token_accuracy": 0.834089708328247,
1076
  "num_tokens": 9769457.0,
1077
  "step": 880
1078
  },
1079
  {
1080
- "entropy": 0.6839869737625122,
1081
  "epoch": 1.7519685039370079,
1082
  "grad_norm": 0.5390625,
1083
  "learning_rate": 7.807367376066889e-05,
1084
- "loss": 0.6296841144561768,
1085
- "mean_token_accuracy": 0.8278504878282547,
1086
  "num_tokens": 9875831.0,
1087
  "step": 890
1088
  },
1089
  {
1090
- "entropy": 0.6922678858041763,
1091
  "epoch": 1.7716535433070866,
1092
- "grad_norm": 0.48046875,
1093
  "learning_rate": 7.60049370040338e-05,
1094
- "loss": 0.6325759887695312,
1095
- "mean_token_accuracy": 0.8210694164037704,
1096
  "num_tokens": 9991758.0,
1097
  "step": 900
1098
  },
1099
  {
1100
  "epoch": 1.7716535433070866,
1101
- "eval_entropy": 0.7464290934580343,
1102
- "eval_loss": 0.8118129372596741,
1103
- "eval_mean_token_accuracy": 0.7863925894101461,
1104
  "eval_num_tokens": 9991758.0,
1105
- "eval_runtime": 15.9375,
1106
- "eval_samples_per_second": 26.792,
1107
- "eval_steps_per_second": 3.388,
1108
  "step": 900
1109
  },
1110
  {
1111
- "entropy": 0.7232977166771889,
1112
  "epoch": 1.7913385826771653,
1113
- "grad_norm": 0.50390625,
1114
  "learning_rate": 7.394704092478742e-05,
1115
- "loss": 0.6871041774749755,
1116
- "mean_token_accuracy": 0.8109384387731552,
1117
  "num_tokens": 10106255.0,
1118
  "step": 910
1119
  },
1120
  {
1121
- "entropy": 0.639159494638443,
1122
  "epoch": 1.811023622047244,
1123
  "grad_norm": 0.51953125,
1124
  "learning_rate": 7.190091525532921e-05,
1125
- "loss": 0.5977220058441162,
1126
- "mean_token_accuracy": 0.8346921026706695,
1127
  "num_tokens": 10218066.0,
1128
  "step": 920
1129
  },
1130
  {
1131
- "entropy": 0.6970785558223724,
1132
  "epoch": 1.8307086614173227,
1133
- "grad_norm": 0.625,
1134
  "learning_rate": 6.98674844103308e-05,
1135
- "loss": 0.6659546852111816,
1136
- "mean_token_accuracy": 0.8177712321281433,
1137
  "num_tokens": 10326991.0,
1138
  "step": 930
1139
  },
1140
  {
1141
- "entropy": 0.705052088201046,
1142
  "epoch": 1.8503937007874016,
1143
- "grad_norm": 0.52734375,
1144
  "learning_rate": 6.784766706909662e-05,
1145
- "loss": 0.6586481094360351,
1146
- "mean_token_accuracy": 0.8198566019535065,
1147
  "num_tokens": 10432558.0,
1148
  "step": 940
1149
  },
1150
  {
1151
- "entropy": 0.6612584158778191,
1152
  "epoch": 1.8700787401574803,
1153
- "grad_norm": 0.65625,
1154
  "learning_rate": 6.58423757605158e-05,
1155
- "loss": 0.6631102561950684,
1156
- "mean_token_accuracy": 0.8250136137008667,
1157
  "num_tokens": 10541702.0,
1158
  "step": 950
1159
  },
1160
  {
1161
  "epoch": 1.8700787401574803,
1162
- "eval_entropy": 0.7582799510823356,
1163
- "eval_loss": 0.8087909817695618,
1164
- "eval_mean_token_accuracy": 0.7858658830324808,
1165
  "eval_num_tokens": 10541702.0,
1166
- "eval_runtime": 15.9386,
1167
- "eval_samples_per_second": 26.79,
1168
- "eval_steps_per_second": 3.388,
1169
  "step": 950
1170
  }
1171
  ],
 
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "entropy": 1.437505340576172,
14
  "epoch": 0.01968503937007874,
15
+ "grad_norm": 0.63671875,
16
  "learning_rate": 3.91304347826087e-05,
17
+ "loss": 1.657536506652832,
18
+ "mean_token_accuracy": 0.6597658038139343,
19
  "num_tokens": 115439.0,
20
  "step": 10
21
  },
22
  {
23
+ "entropy": 1.5124324679374694,
24
  "epoch": 0.03937007874015748,
25
+ "grad_norm": 0.4453125,
26
  "learning_rate": 8.260869565217392e-05,
27
+ "loss": 1.4648273468017579,
28
+ "mean_token_accuracy": 0.683241218328476,
29
  "num_tokens": 225264.0,
30
  "step": 20
31
  },
32
  {
33
+ "entropy": 1.2266616582870484,
34
  "epoch": 0.05905511811023622,
35
  "grad_norm": 0.6796875,
36
  "learning_rate": 0.00012608695652173915,
37
+ "loss": 1.113858699798584,
38
+ "mean_token_accuracy": 0.7449506461620331,
39
  "num_tokens": 330923.0,
40
  "step": 30
41
  },
42
  {
43
+ "entropy": 1.05804543197155,
44
  "epoch": 0.07874015748031496,
45
+ "grad_norm": 0.47265625,
46
  "learning_rate": 0.00016956521739130436,
47
+ "loss": 1.076363468170166,
48
+ "mean_token_accuracy": 0.7455369174480438,
49
  "num_tokens": 437622.0,
50
  "step": 40
51
  },
52
  {
53
+ "entropy": 0.9502384155988693,
54
  "epoch": 0.0984251968503937,
55
+ "grad_norm": 0.5,
56
  "learning_rate": 0.00019999796688511753,
57
+ "loss": 0.9587571144104003,
58
+ "mean_token_accuracy": 0.7683711260557174,
59
  "num_tokens": 551175.0,
60
  "step": 50
61
  },
62
  {
63
  "epoch": 0.0984251968503937,
64
+ "eval_entropy": 1.0894114132280703,
65
+ "eval_loss": 1.0480613708496094,
66
+ "eval_mean_token_accuracy": 0.745736007337217,
67
  "eval_num_tokens": 551175.0,
68
+ "eval_runtime": 15.9708,
69
+ "eval_samples_per_second": 26.736,
70
+ "eval_steps_per_second": 3.381,
71
  "step": 50
72
  },
73
  {
74
+ "entropy": 0.9499408066272735,
75
  "epoch": 0.11811023622047244,
76
+ "grad_norm": 0.5859375,
77
  "learning_rate": 0.0001999618249203163,
78
+ "loss": 0.9569917678833008,
79
+ "mean_token_accuracy": 0.7686831414699554,
80
  "num_tokens": 658006.0,
81
  "step": 60
82
  },
83
  {
84
+ "entropy": 1.0497896373271942,
85
  "epoch": 0.1377952755905512,
86
+ "grad_norm": 0.52734375,
87
  "learning_rate": 0.00019988052141977201,
88
+ "loss": 0.9973426818847656,
89
+ "mean_token_accuracy": 0.7564341962337494,
90
  "num_tokens": 771278.0,
91
  "step": 70
92
  },
93
  {
94
+ "entropy": 0.8503248453140259,
95
  "epoch": 0.15748031496062992,
96
+ "grad_norm": 0.5,
97
  "learning_rate": 0.0001997540931154166,
98
+ "loss": 0.839632225036621,
99
+ "mean_token_accuracy": 0.7869657546281814,
100
  "num_tokens": 886071.0,
101
  "step": 80
102
  },
103
  {
104
+ "entropy": 0.9232195496559144,
105
  "epoch": 0.17716535433070865,
106
+ "grad_norm": 0.482421875,
107
  "learning_rate": 0.00019958259712601903,
108
+ "loss": 0.8961009025573731,
109
+ "mean_token_accuracy": 0.7736886650323868,
110
  "num_tokens": 996361.0,
111
  "step": 90
112
  },
113
  {
114
+ "entropy": 0.8409020006656647,
115
  "epoch": 0.1968503937007874,
116
+ "grad_norm": 0.515625,
117
  "learning_rate": 0.00019936611093137983,
118
+ "loss": 0.839316463470459,
119
+ "mean_token_accuracy": 0.787284591794014,
120
  "num_tokens": 1103999.0,
121
  "step": 100
122
  },
123
  {
124
  "epoch": 0.1968503937007874,
125
+ "eval_entropy": 0.988924115896225,
126
+ "eval_loss": 0.9491098523139954,
127
+ "eval_mean_token_accuracy": 0.7609603978969433,
128
  "eval_num_tokens": 1103999.0,
129
+ "eval_runtime": 15.8986,
130
+ "eval_samples_per_second": 26.858,
131
+ "eval_steps_per_second": 3.397,
132
  "step": 100
133
  },
134
  {
135
+ "entropy": 0.8768177807331086,
136
  "epoch": 0.21653543307086615,
137
+ "grad_norm": 0.41015625,
138
  "learning_rate": 0.00019910473233732668,
139
+ "loss": 0.8348051071166992,
140
+ "mean_token_accuracy": 0.7883853554725647,
141
  "num_tokens": 1216586.0,
142
  "step": 110
143
  },
144
  {
145
+ "entropy": 0.8739962369203568,
146
  "epoch": 0.23622047244094488,
147
+ "grad_norm": 0.59375,
148
  "learning_rate": 0.00019879857943152674,
149
+ "loss": 0.8352558135986328,
150
+ "mean_token_accuracy": 0.7789304822683334,
151
  "num_tokens": 1324767.0,
152
  "step": 120
153
  },
154
  {
155
+ "entropy": 0.8777758985757828,
156
  "epoch": 0.2559055118110236,
157
+ "grad_norm": 0.54296875,
158
  "learning_rate": 0.00019844779053013637,
159
+ "loss": 0.8331022262573242,
160
+ "mean_token_accuracy": 0.7784219890832901,
161
  "num_tokens": 1432356.0,
162
  "step": 130
163
  },
164
  {
165
+ "entropy": 0.7877890735864639,
166
  "epoch": 0.2755905511811024,
167
+ "grad_norm": 0.55078125,
168
  "learning_rate": 0.00019805252411531126,
169
+ "loss": 0.7960677146911621,
170
+ "mean_token_accuracy": 0.7926767230033874,
171
  "num_tokens": 1542545.0,
172
  "step": 140
173
  },
174
  {
175
+ "entropy": 0.8658760845661163,
176
  "epoch": 0.2952755905511811,
177
+ "grad_norm": 0.5625,
178
  "learning_rate": 0.0001976129587636064,
179
+ "loss": 0.8095477104187012,
180
+ "mean_token_accuracy": 0.7904855847358704,
181
  "num_tokens": 1654462.0,
182
  "step": 150
183
  },
184
  {
185
  "epoch": 0.2952755905511811,
186
+ "eval_entropy": 0.9384135085123556,
187
+ "eval_loss": 0.9150573015213013,
188
+ "eval_mean_token_accuracy": 0.7675120333830515,
189
  "eval_num_tokens": 1654462.0,
190
+ "eval_runtime": 15.91,
191
+ "eval_samples_per_second": 26.839,
192
+ "eval_steps_per_second": 3.394,
193
  "step": 150
194
  },
195
  {
196
+ "entropy": 0.771521957218647,
197
  "epoch": 0.31496062992125984,
198
  "grad_norm": 0.470703125,
199
  "learning_rate": 0.00019712929306529723,
200
+ "loss": 0.7662525177001953,
201
+ "mean_token_accuracy": 0.8057839542627334,
202
  "num_tokens": 1760181.0,
203
  "step": 160
204
  },
205
  {
206
+ "entropy": 0.8810683578252793,
207
  "epoch": 0.3346456692913386,
208
  "grad_norm": 0.625,
209
  "learning_rate": 0.000196601745534659,
210
+ "loss": 0.8402512550354004,
211
+ "mean_token_accuracy": 0.7819562494754791,
212
  "num_tokens": 1864683.0,
213
  "step": 170
214
  },
215
  {
216
+ "entropy": 0.8320274949073792,
217
  "epoch": 0.3543307086614173,
218
+ "grad_norm": 0.474609375,
219
  "learning_rate": 0.0001960305545112445,
220
+ "loss": 0.8194713592529297,
221
+ "mean_token_accuracy": 0.7880559414625168,
222
  "num_tokens": 1978641.0,
223
  "step": 180
224
  },
225
  {
226
+ "entropy": 0.8050856739282608,
227
  "epoch": 0.37401574803149606,
228
+ "grad_norm": 0.4921875,
229
  "learning_rate": 0.0001954159780522056,
230
+ "loss": 0.8002090454101562,
231
+ "mean_token_accuracy": 0.791279673576355,
232
  "num_tokens": 2087989.0,
233
  "step": 190
234
  },
235
  {
236
+ "entropy": 0.8930813580751419,
237
  "epoch": 0.3937007874015748,
238
+ "grad_norm": 0.6171875,
239
  "learning_rate": 0.00019475829381570566,
240
+ "loss": 0.8300093650817871,
241
+ "mean_token_accuracy": 0.7832900315523148,
242
  "num_tokens": 2195494.0,
243
  "step": 200
244
  },
245
  {
246
  "epoch": 0.3937007874015748,
247
+ "eval_entropy": 0.9506367754053187,
248
+ "eval_loss": 0.8978024125099182,
249
+ "eval_mean_token_accuracy": 0.7694783839914534,
250
  "eval_num_tokens": 2195494.0,
251
+ "eval_runtime": 15.9025,
252
+ "eval_samples_per_second": 26.851,
253
+ "eval_steps_per_second": 3.396,
254
  "step": 200
255
  },
256
  {
257
+ "entropy": 0.7895667225122451,
258
  "epoch": 0.41338582677165353,
259
+ "grad_norm": 0.41015625,
260
  "learning_rate": 0.0001940577989354773,
261
+ "loss": 0.7946149826049804,
262
+ "mean_token_accuracy": 0.7988882720470428,
263
  "num_tokens": 2314228.0,
264
  "step": 210
265
  },
266
  {
267
+ "entropy": 0.8575397372245789,
268
  "epoch": 0.4330708661417323,
269
+ "grad_norm": 0.6640625,
270
  "learning_rate": 0.00019331480988658083,
271
+ "loss": 0.812718391418457,
272
+ "mean_token_accuracy": 0.7836957842111587,
273
  "num_tokens": 2419122.0,
274
  "step": 220
275
  },
276
  {
277
+ "entropy": 0.8544128388166428,
278
  "epoch": 0.452755905511811,
279
+ "grad_norm": 0.462890625,
280
  "learning_rate": 0.00019252966234242468,
281
+ "loss": 0.8198953628540039,
282
+ "mean_token_accuracy": 0.7827243804931641,
283
  "num_tokens": 2524623.0,
284
  "step": 230
285
  },
286
  {
287
+ "entropy": 0.9001998215913772,
288
  "epoch": 0.47244094488188976,
289
+ "grad_norm": 0.59375,
290
  "learning_rate": 0.00019170271102311238,
291
+ "loss": 0.9015755653381348,
292
+ "mean_token_accuracy": 0.7709000498056412,
293
  "num_tokens": 2631031.0,
294
  "step": 240
295
  },
296
  {
297
+ "entropy": 0.7926925078034401,
298
  "epoch": 0.4921259842519685,
299
+ "grad_norm": 0.49609375,
300
  "learning_rate": 0.00019083432953518418,
301
+ "loss": 0.8107254028320312,
302
+ "mean_token_accuracy": 0.7990583658218384,
303
  "num_tokens": 2741031.0,
304
  "step": 250
305
  },
306
  {
307
  "epoch": 0.4921259842519685,
308
+ "eval_entropy": 0.9023421671655443,
309
+ "eval_loss": 0.884621262550354,
310
+ "eval_mean_token_accuracy": 0.7719593346118927,
311
  "eval_num_tokens": 2741031.0,
312
+ "eval_runtime": 15.8929,
313
+ "eval_samples_per_second": 26.867,
314
+ "eval_steps_per_second": 3.398,
315
  "step": 250
316
  },
317
  {
318
+ "entropy": 0.7738933145999909,
319
  "epoch": 0.5118110236220472,
320
+ "grad_norm": 0.58203125,
321
  "learning_rate": 0.00018992491020282618,
322
+ "loss": 0.7572973251342774,
323
+ "mean_token_accuracy": 0.7949966192245483,
324
  "num_tokens": 2853547.0,
325
  "step": 260
326
  },
327
  {
328
+ "entropy": 0.8569424450397491,
329
  "epoch": 0.531496062992126,
330
+ "grad_norm": 1.375,
331
  "learning_rate": 0.0001889748638906231,
332
+ "loss": 0.8083616256713867,
333
+ "mean_token_accuracy": 0.791676989197731,
334
  "num_tokens": 2968367.0,
335
  "step": 270
336
  },
337
  {
338
+ "entropy": 0.889449554681778,
339
  "epoch": 0.5511811023622047,
340
+ "grad_norm": 0.44921875,
341
  "learning_rate": 0.00018798461981793462,
342
+ "loss": 0.8472497940063477,
343
+ "mean_token_accuracy": 0.7779456943273544,
344
  "num_tokens": 3075963.0,
345
  "step": 280
346
  },
347
  {
348
+ "entropy": 0.8320091620087624,
349
  "epoch": 0.5708661417322834,
350
+ "grad_norm": 0.453125,
351
  "learning_rate": 0.00018695462536497934,
352
+ "loss": 0.8240900993347168,
353
+ "mean_token_accuracy": 0.7846117675304413,
354
  "num_tokens": 3185899.0,
355
  "step": 290
356
  },
357
  {
358
+ "entropy": 0.8333972141146659,
359
  "epoch": 0.5905511811023622,
360
+ "grad_norm": 0.486328125,
361
  "learning_rate": 0.00018588534587071385,
362
+ "loss": 0.8117186546325683,
363
+ "mean_token_accuracy": 0.7908233135938645,
364
  "num_tokens": 3297319.0,
365
  "step": 300
366
  },
367
  {
368
  "epoch": 0.5905511811023622,
369
+ "eval_entropy": 0.8452967874429844,
370
+ "eval_loss": 0.8709425926208496,
371
+ "eval_mean_token_accuracy": 0.7752477537702631,
372
  "eval_num_tokens": 3297319.0,
373
+ "eval_runtime": 15.8962,
374
+ "eval_samples_per_second": 26.862,
375
+ "eval_steps_per_second": 3.397,
376
  "step": 300
377
  },
378
  {
379
+ "entropy": 0.8324314504861832,
380
  "epoch": 0.610236220472441,
381
+ "grad_norm": 0.39453125,
382
  "learning_rate": 0.00018477726442259806,
383
+ "loss": 0.801210880279541,
384
+ "mean_token_accuracy": 0.7885541886091232,
385
  "num_tokens": 3415477.0,
386
  "step": 310
387
  },
388
  {
389
+ "entropy": 0.8624329552054405,
390
  "epoch": 0.6299212598425197,
391
+ "grad_norm": 0.400390625,
392
  "learning_rate": 0.00018363088163834252,
393
+ "loss": 0.8126995086669921,
394
+ "mean_token_accuracy": 0.785911500453949,
395
  "num_tokens": 3524759.0,
396
  "step": 320
397
  },
398
  {
399
+ "entropy": 0.8507865130901336,
400
  "epoch": 0.6496062992125984,
401
  "grad_norm": 0.458984375,
402
  "learning_rate": 0.00018244671543973513,
403
+ "loss": 0.8206624031066895,
404
+ "mean_token_accuracy": 0.7833890795707703,
405
  "num_tokens": 3640838.0,
406
  "step": 330
407
  },
408
  {
409
+ "entropy": 0.7785286724567413,
410
  "epoch": 0.6692913385826772,
411
  "grad_norm": 0.51953125,
412
  "learning_rate": 0.0001812253008186505,
413
+ "loss": 0.753394889831543,
414
+ "mean_token_accuracy": 0.8014622688293457,
415
  "num_tokens": 3756735.0,
416
  "step": 340
417
  },
418
  {
419
+ "entropy": 0.8389557063579559,
420
  "epoch": 0.6889763779527559,
421
+ "grad_norm": 0.50390625,
422
  "learning_rate": 0.0001799671895953471,
423
+ "loss": 0.8116916656494141,
424
+ "mean_token_accuracy": 0.7909021496772766,
425
  "num_tokens": 3870377.0,
426
  "step": 350
427
  },
428
  {
429
  "epoch": 0.6889763779527559,
430
+ "eval_entropy": 0.8793082656683745,
431
+ "eval_loss": 0.8570566177368164,
432
+ "eval_mean_token_accuracy": 0.7760916120476193,
433
  "eval_num_tokens": 3870377.0,
434
+ "eval_runtime": 15.9217,
435
+ "eval_samples_per_second": 26.819,
436
+ "eval_steps_per_second": 3.392,
437
  "step": 350
438
  },
439
  {
440
+ "entropy": 0.7998805820941925,
441
  "epoch": 0.7086614173228346,
442
+ "grad_norm": 0.494140625,
443
  "learning_rate": 0.00017867295016916167,
444
+ "loss": 0.7578703880310058,
445
+ "mean_token_accuracy": 0.7953435629606247,
446
  "num_tokens": 3975692.0,
447
  "step": 360
448
  },
449
  {
450
+ "entropy": 0.9054980486631393,
451
  "epoch": 0.7283464566929134,
452
+ "grad_norm": 0.4765625,
453
  "learning_rate": 0.0001773431672617132,
454
+ "loss": 0.8652525901794433,
455
+ "mean_token_accuracy": 0.7738145262002945,
456
  "num_tokens": 4087256.0,
457
  "step": 370
458
  },
459
  {
460
+ "entropy": 0.7607795268297195,
461
  "epoch": 0.7480314960629921,
462
+ "grad_norm": 0.498046875,
463
  "learning_rate": 0.00017597844165273286,
464
+ "loss": 0.7365350723266602,
465
+ "mean_token_accuracy": 0.7966376423835755,
466
  "num_tokens": 4192629.0,
467
  "step": 380
468
  },
469
  {
470
+ "entropy": 0.8066679537296295,
471
  "epoch": 0.7677165354330708,
472
+ "grad_norm": 0.443359375,
473
  "learning_rate": 0.00017457938990863908,
474
+ "loss": 0.7610409736633301,
475
+ "mean_token_accuracy": 0.8000116467475891,
476
  "num_tokens": 4303838.0,
477
  "step": 390
478
  },
479
  {
480
+ "entropy": 0.8927708208560944,
481
  "epoch": 0.7874015748031497,
482
+ "grad_norm": 0.494140625,
483
  "learning_rate": 0.00017314664410398033,
484
+ "loss": 0.8361794471740722,
485
+ "mean_token_accuracy": 0.7784365385770797,
486
  "num_tokens": 4414002.0,
487
  "step": 400
488
  },
489
  {
490
  "epoch": 0.7874015748031497,
491
+ "eval_entropy": 0.8607326823252218,
492
+ "eval_loss": 0.8459087610244751,
493
+ "eval_mean_token_accuracy": 0.7791097947844753,
494
  "eval_num_tokens": 4414002.0,
495
+ "eval_runtime": 16.176,
496
+ "eval_samples_per_second": 26.397,
497
+ "eval_steps_per_second": 3.338,
498
  "step": 400
499
  },
500
  {
501
+ "entropy": 0.7851713225245476,
502
  "epoch": 0.8070866141732284,
503
+ "grad_norm": 0.56640625,
504
  "learning_rate": 0.0001716808515358714,
505
+ "loss": 0.8022769927978516,
506
+ "mean_token_accuracy": 0.7959282606840133,
507
  "num_tokens": 4525758.0,
508
  "step": 410
509
  },
510
  {
511
+ "entropy": 0.7688719958066941,
512
  "epoch": 0.8267716535433071,
513
+ "grad_norm": 0.447265625,
514
  "learning_rate": 0.00017018267443155268,
515
+ "loss": 0.7182153224945068,
516
+ "mean_token_accuracy": 0.8076630532741547,
517
  "num_tokens": 4640742.0,
518
  "step": 420
519
  },
520
  {
521
+ "entropy": 0.8512372523546219,
522
  "epoch": 0.8464566929133859,
523
+ "grad_norm": 0.455078125,
524
  "learning_rate": 0.00016865278964920378,
525
+ "loss": 0.8275189399719238,
526
+ "mean_token_accuracy": 0.7865870356559753,
527
  "num_tokens": 4754244.0,
528
  "step": 430
529
  },
530
  {
531
+ "entropy": 0.7933165490627289,
532
  "epoch": 0.8661417322834646,
533
+ "grad_norm": 0.34765625,
534
  "learning_rate": 0.00016709188837214732,
535
+ "loss": 0.7769423484802246,
536
+ "mean_token_accuracy": 0.7905291795730591,
537
  "num_tokens": 4862576.0,
538
  "step": 440
539
  },
540
  {
541
+ "entropy": 0.7952004507184028,
542
  "epoch": 0.8858267716535433,
543
+ "grad_norm": 0.375,
544
  "learning_rate": 0.0001655006757965806,
545
+ "loss": 0.755205774307251,
546
+ "mean_token_accuracy": 0.7940250873565674,
547
  "num_tokens": 4984410.0,
548
  "step": 450
549
  },
550
  {
551
  "epoch": 0.8858267716535433,
552
+ "eval_entropy": 0.8390876281040686,
553
+ "eval_loss": 0.8377042412757874,
554
+ "eval_mean_token_accuracy": 0.7800426229282662,
555
  "eval_num_tokens": 4984410.0,
556
+ "eval_runtime": 15.9314,
557
+ "eval_samples_per_second": 26.802,
558
+ "eval_steps_per_second": 3.39,
559
  "step": 450
560
  },
561
  {
562
+ "entropy": 0.8776025518774986,
563
  "epoch": 0.905511811023622,
564
+ "grad_norm": 0.5703125,
565
  "learning_rate": 0.0001638798708129768,
566
+ "loss": 0.8327735900878906,
567
+ "mean_token_accuracy": 0.7833347082138061,
568
  "num_tokens": 5094029.0,
569
  "step": 460
570
  },
571
  {
572
+ "entropy": 0.8570419639348984,
573
  "epoch": 0.9251968503937008,
574
+ "grad_norm": 0.38671875,
575
  "learning_rate": 0.00016223020568129854,
576
+ "loss": 0.830419921875,
577
+ "mean_token_accuracy": 0.7789777547121048,
578
  "num_tokens": 5210613.0,
579
  "step": 470
580
  },
581
  {
582
+ "entropy": 0.8276598662137985,
583
  "epoch": 0.9448818897637795,
584
  "grad_norm": 0.59375,
585
  "learning_rate": 0.00016055242570017244,
586
+ "loss": 0.787315034866333,
587
+ "mean_token_accuracy": 0.7914238631725311,
588
  "num_tokens": 5322546.0,
589
  "step": 480
590
  },
591
  {
592
+ "entropy": 0.8109192818403244,
593
  "epoch": 0.9645669291338582,
594
+ "grad_norm": 0.4375,
595
  "learning_rate": 0.00015884728887017162,
596
+ "loss": 0.7857238292694092,
597
+ "mean_token_accuracy": 0.7958390027284622,
598
  "num_tokens": 5434008.0,
599
  "step": 490
600
  },
601
  {
602
+ "entropy": 0.6643441513180732,
603
  "epoch": 0.984251968503937,
604
+ "grad_norm": 0.419921875,
605
  "learning_rate": 0.0001571155655513606,
606
+ "loss": 0.6719898700714111,
607
+ "mean_token_accuracy": 0.8172835201025009,
608
  "num_tokens": 5544086.0,
609
  "step": 500
610
  },
611
  {
612
  "epoch": 0.984251968503937,
613
+ "eval_entropy": 0.8121603627999624,
614
+ "eval_loss": 0.8310334086418152,
615
+ "eval_mean_token_accuracy": 0.7817157308260599,
616
  "eval_num_tokens": 5544086.0,
617
+ "eval_runtime": 15.9111,
618
+ "eval_samples_per_second": 26.837,
619
+ "eval_steps_per_second": 3.394,
620
  "step": 500
621
  },
622
  {
623
+ "entropy": 0.7330836996436119,
624
  "epoch": 1.0039370078740157,
625
+ "grad_norm": 0.38671875,
626
  "learning_rate": 0.0001553580381152559,
627
+ "loss": 0.6914227962493896,
628
+ "mean_token_accuracy": 0.8179270803928376,
629
  "num_tokens": 5657246.0,
630
  "step": 510
631
  },
632
  {
633
+ "entropy": 0.7551421344280242,
634
  "epoch": 1.0236220472440944,
635
+ "grad_norm": 0.486328125,
636
  "learning_rate": 0.0001535755005913602,
637
+ "loss": 0.6854944705963135,
638
+ "mean_token_accuracy": 0.8148082345724106,
639
  "num_tokens": 5768184.0,
640
  "step": 520
641
  },
642
  {
643
+ "entropy": 0.7464401632547378,
644
  "epoch": 1.0433070866141732,
645
+ "grad_norm": 0.6015625,
646
  "learning_rate": 0.00015176875830842938,
647
+ "loss": 0.7031197071075439,
648
+ "mean_token_accuracy": 0.814039769768715,
649
  "num_tokens": 5885598.0,
650
  "step": 530
651
  },
652
  {
653
+ "entropy": 0.6804196655750274,
654
  "epoch": 1.0629921259842519,
655
+ "grad_norm": 0.435546875,
656
  "learning_rate": 0.0001499386275306348,
657
+ "loss": 0.6404387950897217,
658
+ "mean_token_accuracy": 0.8235772669315338,
659
  "num_tokens": 6002120.0,
660
  "step": 540
661
  },
662
  {
663
+ "entropy": 0.6835484370589257,
664
  "epoch": 1.0826771653543308,
665
+ "grad_norm": 0.416015625,
666
  "learning_rate": 0.0001480859350887852,
667
+ "loss": 0.6457780361175537,
668
+ "mean_token_accuracy": 0.8242624312639236,
669
  "num_tokens": 6112085.0,
670
  "step": 550
671
  },
672
  {
673
  "epoch": 1.0826771653543308,
674
+ "eval_entropy": 0.7967574977212482,
675
+ "eval_loss": 0.835203230381012,
676
+ "eval_mean_token_accuracy": 0.7809422369356509,
677
  "eval_num_tokens": 6112085.0,
678
+ "eval_runtime": 15.9214,
679
+ "eval_samples_per_second": 26.819,
680
+ "eval_steps_per_second": 3.392,
681
  "step": 550
682
  },
683
  {
684
+ "entropy": 0.7008466422557831,
685
  "epoch": 1.1023622047244095,
686
+ "grad_norm": 0.48046875,
687
  "learning_rate": 0.00014621151800677434,
688
+ "loss": 0.6664916515350342,
689
+ "mean_token_accuracy": 0.8154505103826523,
690
  "num_tokens": 6224645.0,
691
  "step": 560
692
  },
693
  {
694
+ "entropy": 0.6077279314398766,
695
  "epoch": 1.1220472440944882,
696
  "grad_norm": 0.37890625,
697
  "learning_rate": 0.0001443162231234242,
698
+ "loss": 0.5706528663635254,
699
+ "mean_token_accuracy": 0.8392371147871017,
700
  "num_tokens": 6336790.0,
701
  "step": 570
702
  },
703
  {
704
+ "entropy": 0.683767618238926,
705
  "epoch": 1.141732283464567,
706
  "grad_norm": 0.55078125,
707
  "learning_rate": 0.0001424009067098931,
708
+ "loss": 0.6489217758178711,
709
+ "mean_token_accuracy": 0.8194159388542175,
710
  "num_tokens": 6450761.0,
711
  "step": 580
712
  },
713
  {
714
+ "entropy": 0.6302935466170311,
715
  "epoch": 1.1614173228346456,
716
+ "grad_norm": 0.486328125,
717
  "learning_rate": 0.00014046643408282305,
718
+ "loss": 0.5900890350341796,
719
+ "mean_token_accuracy": 0.8340148836374283,
720
  "num_tokens": 6562060.0,
721
  "step": 590
722
  },
723
  {
724
+ "entropy": 0.6183161422610283,
725
  "epoch": 1.1811023622047245,
726
+ "grad_norm": 0.455078125,
727
  "learning_rate": 0.00013851367921340017,
728
+ "loss": 0.5828392982482911,
729
+ "mean_token_accuracy": 0.835411223769188,
730
  "num_tokens": 6676922.0,
731
  "step": 600
732
  },
733
  {
734
  "epoch": 1.1811023622047245,
735
+ "eval_entropy": 0.7808757076660792,
736
+ "eval_loss": 0.835366427898407,
737
+ "eval_mean_token_accuracy": 0.7813083474282865,
738
  "eval_num_tokens": 6676922.0,
739
+ "eval_runtime": 15.8899,
740
+ "eval_samples_per_second": 26.872,
741
+ "eval_steps_per_second": 3.398,
742
  "step": 600
743
  },
744
  {
745
+ "entropy": 0.6484389528632164,
746
  "epoch": 1.2007874015748032,
747
+ "grad_norm": 0.4921875,
748
  "learning_rate": 0.0001365435243325051,
749
+ "loss": 0.6237234592437744,
750
+ "mean_token_accuracy": 0.8273959398269654,
751
  "num_tokens": 6783497.0,
752
  "step": 610
753
  },
754
  {
755
+ "entropy": 0.6731782004237175,
756
  "epoch": 1.220472440944882,
757
+ "grad_norm": 0.54296875,
758
  "learning_rate": 0.00013455685953213194,
759
+ "loss": 0.6462783336639404,
760
+ "mean_token_accuracy": 0.8242785036563873,
761
  "num_tokens": 6894014.0,
762
  "step": 620
763
  },
764
  {
765
+ "entropy": 0.6746662899851799,
766
  "epoch": 1.2401574803149606,
767
+ "grad_norm": 0.484375,
768
  "learning_rate": 0.0001325545823632554,
769
+ "loss": 0.6090981006622315,
770
+ "mean_token_accuracy": 0.8258850336074829,
771
  "num_tokens": 6997350.0,
772
  "step": 630
773
  },
774
  {
775
+ "entropy": 0.6707195937633514,
776
  "epoch": 1.2598425196850394,
777
  "grad_norm": 0.451171875,
778
  "learning_rate": 0.00013053759743032823,
779
+ "loss": 0.6500792026519775,
780
+ "mean_token_accuracy": 0.8198623776435852,
781
  "num_tokens": 7109067.0,
782
  "step": 640
783
  },
784
  {
785
+ "entropy": 0.7742280542850495,
786
  "epoch": 1.279527559055118,
787
+ "grad_norm": 0.55859375,
788
  "learning_rate": 0.00012850681598259195,
789
+ "loss": 0.717244005203247,
790
+ "mean_token_accuracy": 0.7994646310806275,
791
  "num_tokens": 7218739.0,
792
  "step": 650
793
  },
794
  {
795
  "epoch": 1.279527559055118,
796
+ "eval_entropy": 0.7477047746932065,
797
+ "eval_loss": 0.8298369646072388,
798
+ "eval_mean_token_accuracy": 0.7822379061469326,
799
  "eval_num_tokens": 7218739.0,
800
+ "eval_runtime": 15.8838,
801
+ "eval_samples_per_second": 26.883,
802
+ "eval_steps_per_second": 3.4,
803
  "step": 650
804
  },
805
  {
806
+ "entropy": 0.7066994801163673,
807
  "epoch": 1.2992125984251968,
808
+ "grad_norm": 0.52734375,
809
  "learning_rate": 0.0001264631555023856,
810
+ "loss": 0.6610036373138428,
811
+ "mean_token_accuracy": 0.8158259451389313,
812
  "num_tokens": 7334561.0,
813
  "step": 660
814
  },
815
  {
816
+ "entropy": 0.6165548652410507,
817
  "epoch": 1.3188976377952755,
818
+ "grad_norm": 0.76953125,
819
  "learning_rate": 0.00012440753929063826,
820
+ "loss": 0.6015829563140869,
821
+ "mean_token_accuracy": 0.8280171245336533,
822
  "num_tokens": 7443871.0,
823
  "step": 670
824
  },
825
  {
826
+ "entropy": 0.7171120464801788,
827
  "epoch": 1.3385826771653544,
828
+ "grad_norm": 0.51171875,
829
  "learning_rate": 0.00012234089604973303,
830
+ "loss": 0.6697704792022705,
831
+ "mean_token_accuracy": 0.816604706645012,
832
  "num_tokens": 7555559.0,
833
  "step": 680
834
  },
835
  {
836
+ "entropy": 0.6375228509306907,
837
  "epoch": 1.358267716535433,
838
+ "grad_norm": 0.51171875,
839
  "learning_rate": 0.0001202641594639305,
840
+ "loss": 0.6138493061065674,
841
+ "mean_token_accuracy": 0.8299814134836196,
842
  "num_tokens": 7669861.0,
843
  "step": 690
844
  },
845
  {
846
+ "entropy": 0.7527845561504364,
847
  "epoch": 1.3779527559055118,
848
+ "grad_norm": 0.54296875,
849
  "learning_rate": 0.00011817826777754171,
850
+ "loss": 0.6848691940307617,
851
+ "mean_token_accuracy": 0.8056917071342469,
852
  "num_tokens": 7782831.0,
853
  "step": 700
854
  },
855
  {
856
  "epoch": 1.3779527559055118,
857
+ "eval_entropy": 0.7785312417480681,
858
+ "eval_loss": 0.822371244430542,
859
+ "eval_mean_token_accuracy": 0.7844836800186722,
860
  "eval_num_tokens": 7782831.0,
861
+ "eval_runtime": 15.9232,
862
+ "eval_samples_per_second": 26.816,
863
+ "eval_steps_per_second": 3.391,
864
  "step": 700
865
  },
866
  {
867
+ "entropy": 0.7150849819183349,
868
  "epoch": 1.3976377952755905,
869
+ "grad_norm": 0.419921875,
870
  "learning_rate": 0.00011608416337104065,
871
+ "loss": 0.6717889785766602,
872
+ "mean_token_accuracy": 0.8194838553667069,
873
  "num_tokens": 7895511.0,
874
  "step": 710
875
  },
876
  {
877
+ "entropy": 0.6417959302663803,
878
  "epoch": 1.4173228346456692,
879
+ "grad_norm": 0.44140625,
880
  "learning_rate": 0.00011398279233530829,
881
+ "loss": 0.6370071411132813,
882
+ "mean_token_accuracy": 0.8262952893972397,
883
  "num_tokens": 8006186.0,
884
  "step": 720
885
  },
886
  {
887
+ "entropy": 0.7074607491493226,
888
  "epoch": 1.4370078740157481,
889
+ "grad_norm": 0.515625,
890
  "learning_rate": 0.00011187510404420028,
891
+ "loss": 0.6439510822296143,
892
+ "mean_token_accuracy": 0.8158842891454696,
893
  "num_tokens": 8110444.0,
894
  "step": 730
895
  },
896
  {
897
+ "entropy": 0.7658031359314919,
898
  "epoch": 1.4566929133858268,
899
  "grad_norm": 0.7109375,
900
  "learning_rate": 0.00010976205072563132,
901
+ "loss": 0.7316959857940674,
902
+ "mean_token_accuracy": 0.8061666846275329,
903
  "num_tokens": 8220128.0,
904
  "step": 740
905
  },
906
  {
907
+ "entropy": 0.6396550685167313,
908
  "epoch": 1.4763779527559056,
909
  "grad_norm": 0.5078125,
910
  "learning_rate": 0.00010764458703137024,
911
+ "loss": 0.6169118881225586,
912
+ "mean_token_accuracy": 0.8309739202260971,
913
  "num_tokens": 8327961.0,
914
  "step": 750
915
  },
916
  {
917
  "epoch": 1.4763779527559056,
918
+ "eval_entropy": 0.7342568177867819,
919
+ "eval_loss": 0.8272306323051453,
920
+ "eval_mean_token_accuracy": 0.7827356656392416,
921
  "eval_num_tokens": 8327961.0,
922
+ "eval_runtime": 15.911,
923
+ "eval_samples_per_second": 26.837,
924
+ "eval_steps_per_second": 3.394,
925
  "step": 750
926
  },
927
  {
928
+ "entropy": 0.6999662488698959,
929
  "epoch": 1.4960629921259843,
930
+ "grad_norm": 0.671875,
931
  "learning_rate": 0.00010552366960573985,
932
+ "loss": 0.6717596054077148,
933
+ "mean_token_accuracy": 0.8167526543140411,
934
  "num_tokens": 8432725.0,
935
  "step": 760
936
  },
937
  {
938
+ "entropy": 0.6736484602093696,
939
  "epoch": 1.515748031496063,
940
+ "grad_norm": 0.408203125,
941
  "learning_rate": 0.00010340025665341684,
942
+ "loss": 0.6454797744750976,
943
+ "mean_token_accuracy": 0.8241631388664246,
944
  "num_tokens": 8550042.0,
945
  "step": 770
946
  },
947
  {
948
+ "entropy": 0.6505535006523132,
949
  "epoch": 1.5354330708661417,
950
+ "grad_norm": 0.45703125,
951
  "learning_rate": 0.00010127530750652642,
952
+ "loss": 0.6001406669616699,
953
+ "mean_token_accuracy": 0.8359915494918824,
954
  "num_tokens": 8664499.0,
955
  "step": 780
956
  },
957
  {
958
+ "entropy": 0.643044576048851,
959
  "epoch": 1.5551181102362204,
960
+ "grad_norm": 0.6875,
961
  "learning_rate": 9.914978219122791e-05,
962
+ "loss": 0.6050069808959961,
963
+ "mean_token_accuracy": 0.8284690707921982,
964
  "num_tokens": 8778459.0,
965
  "step": 790
966
  },
967
  {
968
+ "entropy": 0.6357371553778648,
969
  "epoch": 1.574803149606299,
970
  "grad_norm": 0.6484375,
971
  "learning_rate": 9.702464099398632e-05,
972
+ "loss": 0.6026933193206787,
973
+ "mean_token_accuracy": 0.8313924312591553,
974
  "num_tokens": 8886361.0,
975
  "step": 800
976
  },
977
  {
978
  "epoch": 1.574803149606299,
979
+ "eval_entropy": 0.7611764465217237,
980
+ "eval_loss": 0.8170427083969116,
981
+ "eval_mean_token_accuracy": 0.7845815287695991,
982
  "eval_num_tokens": 8886361.0,
983
+ "eval_runtime": 15.9048,
984
+ "eval_samples_per_second": 26.847,
985
+ "eval_steps_per_second": 3.395,
986
  "step": 800
987
  },
988
  {
989
+ "entropy": 0.6775295034050941,
990
  "epoch": 1.594488188976378,
991
  "grad_norm": 0.515625,
992
  "learning_rate": 9.4900844027727e-05,
993
+ "loss": 0.6273195743560791,
994
+ "mean_token_accuracy": 0.8267990559339523,
995
  "num_tokens": 8995558.0,
996
  "step": 810
997
  },
998
  {
999
+ "entropy": 0.7035147309303283,
1000
  "epoch": 1.6141732283464567,
1001
+ "grad_norm": 0.53515625,
1002
  "learning_rate": 9.277935079806795e-05,
1003
+ "loss": 0.6532655715942383,
1004
+ "mean_token_accuracy": 0.8209419757127762,
1005
  "num_tokens": 9096245.0,
1006
  "step": 820
1007
  },
1008
  {
1009
+ "entropy": 0.6083378851413727,
1010
  "epoch": 1.6338582677165354,
1011
+ "grad_norm": 0.796875,
1012
  "learning_rate": 9.066111976982712e-05,
1013
+ "loss": 0.5965664386749268,
1014
+ "mean_token_accuracy": 0.8344648003578186,
1015
  "num_tokens": 9204264.0,
1016
  "step": 830
1017
  },
1018
  {
1019
+ "entropy": 0.6942630469799042,
1020
  "epoch": 1.6535433070866141,
1021
+ "grad_norm": 0.50390625,
1022
  "learning_rate": 8.854710793399947e-05,
1023
+ "loss": 0.6506452560424805,
1024
+ "mean_token_accuracy": 0.8177429378032685,
1025
  "num_tokens": 9314940.0,
1026
  "step": 840
1027
  },
1028
  {
1029
+ "entropy": 0.6778671249747277,
1030
  "epoch": 1.673228346456693,
1031
  "grad_norm": 0.546875,
1032
  "learning_rate": 8.64382703754003e-05,
1033
+ "loss": 0.6361007213592529,
1034
+ "mean_token_accuracy": 0.8213603168725967,
1035
  "num_tokens": 9429496.0,
1036
  "step": 850
1037
  },
1038
  {
1039
  "epoch": 1.673228346456693,
1040
+ "eval_entropy": 0.7621777179064574,
1041
+ "eval_loss": 0.8149666786193848,
1042
+ "eval_mean_token_accuracy": 0.7850644290447235,
1043
  "eval_num_tokens": 9429496.0,
1044
+ "eval_runtime": 15.9323,
1045
+ "eval_samples_per_second": 26.801,
1046
+ "eval_steps_per_second": 3.389,
1047
  "step": 850
1048
  },
1049
  {
1050
+ "entropy": 0.6798793867230415,
1051
  "epoch": 1.6929133858267718,
1052
+ "grad_norm": 0.46875,
1053
  "learning_rate": 8.433555984116937e-05,
1054
+ "loss": 0.613386869430542,
1055
+ "mean_token_accuracy": 0.8275126874446869,
1056
  "num_tokens": 9544270.0,
1057
  "step": 860
1058
  },
1059
  {
1060
+ "entropy": 0.6023602589964867,
1061
  "epoch": 1.7125984251968505,
1062
  "grad_norm": 0.53125,
1063
  "learning_rate": 8.22399263103314e-05,
1064
+ "loss": 0.6046566963195801,
1065
+ "mean_token_accuracy": 0.832113066315651,
1066
  "num_tokens": 9654198.0,
1067
  "step": 870
1068
  },
1069
  {
1070
+ "entropy": 0.640959857404232,
1071
  "epoch": 1.7322834645669292,
1072
+ "grad_norm": 0.6875,
1073
  "learning_rate": 8.015231656460712e-05,
1074
+ "loss": 0.6335668563842773,
1075
+ "mean_token_accuracy": 0.8338181763887406,
1076
  "num_tokens": 9769457.0,
1077
  "step": 880
1078
  },
1079
  {
1080
+ "entropy": 0.6831659495830535,
1081
  "epoch": 1.7519685039370079,
1082
  "grad_norm": 0.5390625,
1083
  "learning_rate": 7.807367376066889e-05,
1084
+ "loss": 0.6274240970611572,
1085
+ "mean_token_accuracy": 0.8270208179950714,
1086
  "num_tokens": 9875831.0,
1087
  "step": 890
1088
  },
1089
  {
1090
+ "entropy": 0.691785454750061,
1091
  "epoch": 1.7716535433070866,
1092
+ "grad_norm": 0.48828125,
1093
  "learning_rate": 7.60049370040338e-05,
1094
+ "loss": 0.631044578552246,
1095
+ "mean_token_accuracy": 0.8213751494884491,
1096
  "num_tokens": 9991758.0,
1097
  "step": 900
1098
  },
1099
  {
1100
  "epoch": 1.7716535433070866,
1101
+ "eval_entropy": 0.741917089731605,
1102
+ "eval_loss": 0.8126867413520813,
1103
+ "eval_mean_token_accuracy": 0.7860870074342798,
1104
  "eval_num_tokens": 9991758.0,
1105
+ "eval_runtime": 15.9328,
1106
+ "eval_samples_per_second": 26.8,
1107
+ "eval_steps_per_second": 3.389,
1108
  "step": 900
1109
  },
1110
  {
1111
+ "entropy": 0.7206486001610756,
1112
  "epoch": 1.7913385826771653,
1113
+ "grad_norm": 0.51171875,
1114
  "learning_rate": 7.394704092478742e-05,
1115
+ "loss": 0.685962438583374,
1116
+ "mean_token_accuracy": 0.8114332675933837,
1117
  "num_tokens": 10106255.0,
1118
  "step": 910
1119
  },
1120
  {
1121
+ "entropy": 0.6408105552196502,
1122
  "epoch": 1.811023622047244,
1123
  "grad_norm": 0.51953125,
1124
  "learning_rate": 7.190091525532921e-05,
1125
+ "loss": 0.5981018543243408,
1126
+ "mean_token_accuracy": 0.8346310049295426,
1127
  "num_tokens": 10218066.0,
1128
  "step": 920
1129
  },
1130
  {
1131
+ "entropy": 0.6974071592092514,
1132
  "epoch": 1.8307086614173227,
1133
+ "grad_norm": 0.62109375,
1134
  "learning_rate": 6.98674844103308e-05,
1135
+ "loss": 0.6660090446472168,
1136
+ "mean_token_accuracy": 0.8166637867689133,
1137
  "num_tokens": 10326991.0,
1138
  "step": 930
1139
  },
1140
  {
1141
+ "entropy": 0.7043072924017906,
1142
  "epoch": 1.8503937007874016,
1143
+ "grad_norm": 0.515625,
1144
  "learning_rate": 6.784766706909662e-05,
1145
+ "loss": 0.6577100276947021,
1146
+ "mean_token_accuracy": 0.8193566143512726,
1147
  "num_tokens": 10432558.0,
1148
  "step": 940
1149
  },
1150
  {
1151
+ "entropy": 0.6612386792898178,
1152
  "epoch": 1.8700787401574803,
1153
+ "grad_norm": 0.66015625,
1154
  "learning_rate": 6.58423757605158e-05,
1155
+ "loss": 0.663103723526001,
1156
+ "mean_token_accuracy": 0.8252974808216095,
1157
  "num_tokens": 10541702.0,
1158
  "step": 950
1159
  },
1160
  {
1161
  "epoch": 1.8700787401574803,
1162
+ "eval_entropy": 0.7594050356635341,
1163
+ "eval_loss": 0.809001624584198,
1164
+ "eval_mean_token_accuracy": 0.7858329779571958,
1165
  "eval_num_tokens": 10541702.0,
1166
+ "eval_runtime": 15.9266,
1167
+ "eval_samples_per_second": 26.811,
1168
+ "eval_steps_per_second": 3.391,
1169
  "step": 950
1170
  }
1171
  ],