Hugging Face's logo

loris3
/
stratified_10m_curriculum_roberta_mattr_increasing

Fill-Mask
Transformers
Safetensors
roberta
Model card Files
xet
 Community
loris3 commited on
Commit
e344451
·
verified ·
1 Parent(s): 967cad1

Upload folder using huggingface_hub

Browse files
Files changed (7) hide show
  1. checkpoints/checkpoint-3132/config.json +26 -0
  2. checkpoints/checkpoint-3132/model.safetensors +3 -0
  3. checkpoints/checkpoint-3132/optimizer.pt +3 -0
  4. checkpoints/checkpoint-3132/rng_state.pth +3 -0
  5. checkpoints/checkpoint-3132/scheduler.pt +3 -0
  6. checkpoints/checkpoint-3132/trainer_state.json +527 -0
  7. checkpoints/checkpoint-3132/training_args.bin +3 -0
checkpoints/checkpoint-3132/config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "RobertaForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "bos_token_id": 0,
7
+ "classifier_dropout": null,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 768,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 3072,
14
+ "layer_norm_eps": 1e-05,
15
+ "max_position_embeddings": 514,
16
+ "model_type": "roberta",
17
+ "num_attention_heads": 12,
18
+ "num_hidden_layers": 12,
19
+ "pad_token_id": 1,
20
+ "position_embedding_type": "absolute",
21
+ "torch_dtype": "float32",
22
+ "transformers_version": "4.47.0",
23
+ "type_vocab_size": 1,
24
+ "use_cache": true,
25
+ "vocab_size": 52000
26
+ }
checkpoints/checkpoint-3132/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7500f8e011bd4e304283cdae6d01edea6aeb281ab72e75cdefc3d88053f191d
3
+ size 504150808
checkpoints/checkpoint-3132/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30fc1c539dd94161d2f17077221ba937d1049a32ca04a8881a4a5a8d14b083f3
3
+ size 1008422138
checkpoints/checkpoint-3132/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46d50c7b965bf44ea6c9efe4c1a860ced53a092ce14783121ca45f40c5931d23
3
+ size 14244
checkpoints/checkpoint-3132/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f865c70ee524959ed1cbeb77883f1c0c001bd7f1447dd1a4058869c67bbd58a6
3
+ size 1064
checkpoints/checkpoint-3132/trainer_state.json ADDED
@@ -0,0 +1,527 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 5.998804113848362,
5
+ "eval_steps": 500,
6
+ "global_step": 3132,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.09567089213106912,
13
+ "grad_norm": 52.747867584228516,
14
+ "learning_rate": 2.5e-06,
15
+ "loss": 10.8165,
16
+ "step": 50
17
+ },
18
+ {
19
+ "epoch": 0.19134178426213824,
20
+ "grad_norm": 66.05134582519531,
21
+ "learning_rate": 5e-06,
22
+ "loss": 10.054,
23
+ "step": 100
24
+ },
25
+ {
26
+ "epoch": 0.2870126763932074,
27
+ "grad_norm": 53.76071548461914,
28
+ "learning_rate": 7.5e-06,
29
+ "loss": 9.0799,
30
+ "step": 150
31
+ },
32
+ {
33
+ "epoch": 0.3826835685242765,
34
+ "grad_norm": 37.95576477050781,
35
+ "learning_rate": 1e-05,
36
+ "loss": 9.0591,
37
+ "step": 200
38
+ },
39
+ {
40
+ "epoch": 0.4783544606553456,
41
+ "grad_norm": 44.30142593383789,
42
+ "learning_rate": 1.25e-05,
43
+ "loss": 9.0954,
44
+ "step": 250
45
+ },
46
+ {
47
+ "epoch": 0.5740253527864148,
48
+ "grad_norm": 44.76091766357422,
49
+ "learning_rate": 1.5e-05,
50
+ "loss": 8.2617,
51
+ "step": 300
52
+ },
53
+ {
54
+ "epoch": 0.6696962449174838,
55
+ "grad_norm": 36.74516677856445,
56
+ "learning_rate": 1.7500000000000002e-05,
57
+ "loss": 7.6069,
58
+ "step": 350
59
+ },
60
+ {
61
+ "epoch": 0.765367137048553,
62
+ "grad_norm": 42.63846206665039,
63
+ "learning_rate": 2e-05,
64
+ "loss": 7.278,
65
+ "step": 400
66
+ },
67
+ {
68
+ "epoch": 0.8610380291796221,
69
+ "grad_norm": 26.08951759338379,
70
+ "learning_rate": 2.2499999999999998e-05,
71
+ "loss": 7.0434,
72
+ "step": 450
73
+ },
74
+ {
75
+ "epoch": 0.9567089213106912,
76
+ "grad_norm": 23.093669891357422,
77
+ "learning_rate": 2.5e-05,
78
+ "loss": 7.5574,
79
+ "step": 500
80
+ },
81
+ {
82
+ "epoch": 0.9988041138483617,
83
+ "eval_accuracy": 0.0,
84
+ "eval_loss": 7.002418041229248,
85
+ "eval_normalizer": 94012.0,
86
+ "eval_runtime": 93.6448,
87
+ "eval_samples_per_second": 570.849,
88
+ "eval_steps_per_second": 1.121,
89
+ "step": 522
90
+ },
91
+ {
92
+ "epoch": 1.0535756995933987,
93
+ "grad_norm": 21.285545349121094,
94
+ "learning_rate": 2.75e-05,
95
+ "loss": 6.9275,
96
+ "step": 550
97
+ },
98
+ {
99
+ "epoch": 1.1492465917244679,
100
+ "grad_norm": 33.146881103515625,
101
+ "learning_rate": 3e-05,
102
+ "loss": 6.8179,
103
+ "step": 600
104
+ },
105
+ {
106
+ "epoch": 1.244917483855537,
107
+ "grad_norm": 28.52762222290039,
108
+ "learning_rate": 3.2500000000000004e-05,
109
+ "loss": 5.2322,
110
+ "step": 650
111
+ },
112
+ {
113
+ "epoch": 1.3405883759866062,
114
+ "grad_norm": 33.8976936340332,
115
+ "learning_rate": 3.5000000000000004e-05,
116
+ "loss": 5.0881,
117
+ "step": 700
118
+ },
119
+ {
120
+ "epoch": 1.4362592681176751,
121
+ "grad_norm": 31.849340438842773,
122
+ "learning_rate": 3.75e-05,
123
+ "loss": 6.3516,
124
+ "step": 750
125
+ },
126
+ {
127
+ "epoch": 1.5319301602487443,
128
+ "grad_norm": 26.819744110107422,
129
+ "learning_rate": 4e-05,
130
+ "loss": 5.6243,
131
+ "step": 800
132
+ },
133
+ {
134
+ "epoch": 1.6276010523798135,
135
+ "grad_norm": 27.690715789794922,
136
+ "learning_rate": 4.25e-05,
137
+ "loss": 5.4867,
138
+ "step": 850
139
+ },
140
+ {
141
+ "epoch": 1.7232719445108826,
142
+ "grad_norm": 39.06813430786133,
143
+ "learning_rate": 4.4999999999999996e-05,
144
+ "loss": 5.3091,
145
+ "step": 900
146
+ },
147
+ {
148
+ "epoch": 1.8189428366419516,
149
+ "grad_norm": 25.289762496948242,
150
+ "learning_rate": 4.75e-05,
151
+ "loss": 5.9413,
152
+ "step": 950
153
+ },
154
+ {
155
+ "epoch": 1.9146137287730207,
156
+ "grad_norm": 31.576900482177734,
157
+ "learning_rate": 5e-05,
158
+ "loss": 6.3911,
159
+ "step": 1000
160
+ },
161
+ {
162
+ "epoch": 1.9988041138483617,
163
+ "eval_accuracy": 0.0,
164
+ "eval_loss": 6.082557201385498,
165
+ "eval_normalizer": 93558.0,
166
+ "eval_runtime": 86.1348,
167
+ "eval_samples_per_second": 620.62,
168
+ "eval_steps_per_second": 1.219,
169
+ "step": 1044
170
+ },
171
+ {
172
+ "epoch": 2.0114805070557282,
173
+ "grad_norm": 36.15195846557617,
174
+ "learning_rate": 5.25e-05,
175
+ "loss": 6.4163,
176
+ "step": 1050
177
+ },
178
+ {
179
+ "epoch": 2.1071513991867974,
180
+ "grad_norm": 28.969654083251953,
181
+ "learning_rate": 5.5e-05,
182
+ "loss": 5.9146,
183
+ "step": 1100
184
+ },
185
+ {
186
+ "epoch": 2.2028222913178666,
187
+ "grad_norm": 31.072589874267578,
188
+ "learning_rate": 5.75e-05,
189
+ "loss": 4.9223,
190
+ "step": 1150
191
+ },
192
+ {
193
+ "epoch": 2.2984931834489357,
194
+ "grad_norm": 33.17365646362305,
195
+ "learning_rate": 6e-05,
196
+ "loss": 3.9321,
197
+ "step": 1200
198
+ },
199
+ {
200
+ "epoch": 2.394164075580005,
201
+ "grad_norm": 20.012985229492188,
202
+ "learning_rate": 6.25e-05,
203
+ "loss": 5.1453,
204
+ "step": 1250
205
+ },
206
+ {
207
+ "epoch": 2.489834967711074,
208
+ "grad_norm": 33.7502326965332,
209
+ "learning_rate": 6.500000000000001e-05,
210
+ "loss": 4.8,
211
+ "step": 1300
212
+ },
213
+ {
214
+ "epoch": 2.5855058598421428,
215
+ "grad_norm": 32.18284225463867,
216
+ "learning_rate": 6.75e-05,
217
+ "loss": 4.5708,
218
+ "step": 1350
219
+ },
220
+ {
221
+ "epoch": 2.6811767519732124,
222
+ "grad_norm": 26.967090606689453,
223
+ "learning_rate": 7.000000000000001e-05,
224
+ "loss": 4.479,
225
+ "step": 1400
226
+ },
227
+ {
228
+ "epoch": 2.776847644104281,
229
+ "grad_norm": 21.180952072143555,
230
+ "learning_rate": 7.25e-05,
231
+ "loss": 4.847,
232
+ "step": 1450
233
+ },
234
+ {
235
+ "epoch": 2.8725185362353503,
236
+ "grad_norm": 34.09960174560547,
237
+ "learning_rate": 7.5e-05,
238
+ "loss": 5.2215,
239
+ "step": 1500
240
+ },
241
+ {
242
+ "epoch": 2.9681894283664194,
243
+ "grad_norm": 30.934101104736328,
244
+ "learning_rate": 7.75e-05,
245
+ "loss": 6.1036,
246
+ "step": 1550
247
+ },
248
+ {
249
+ "epoch": 2.9988041138483617,
250
+ "eval_accuracy": 0.0,
251
+ "eval_loss": 5.322023868560791,
252
+ "eval_normalizer": 94216.0,
253
+ "eval_runtime": 101.6167,
254
+ "eval_samples_per_second": 526.065,
255
+ "eval_steps_per_second": 1.033,
256
+ "step": 1566
257
+ },
258
+ {
259
+ "epoch": 3.065056206649127,
260
+ "grad_norm": 25.038644790649414,
261
+ "learning_rate": 8e-05,
262
+ "loss": 5.2087,
263
+ "step": 1600
264
+ },
265
+ {
266
+ "epoch": 3.160727098780196,
267
+ "grad_norm": 32.0327033996582,
268
+ "learning_rate": 8.25e-05,
269
+ "loss": 5.0918,
270
+ "step": 1650
271
+ },
272
+ {
273
+ "epoch": 3.2563979909112652,
274
+ "grad_norm": 33.0177001953125,
275
+ "learning_rate": 8.5e-05,
276
+ "loss": 3.4156,
277
+ "step": 1700
278
+ },
279
+ {
280
+ "epoch": 3.3520688830423344,
281
+ "grad_norm": 24.040756225585938,
282
+ "learning_rate": 8.75e-05,
283
+ "loss": 3.7571,
284
+ "step": 1750
285
+ },
286
+ {
287
+ "epoch": 3.4477397751734036,
288
+ "grad_norm": 27.80860710144043,
289
+ "learning_rate": 8.999999999999999e-05,
290
+ "loss": 4.6411,
291
+ "step": 1800
292
+ },
293
+ {
294
+ "epoch": 3.5434106673044727,
295
+ "grad_norm": 25.351137161254883,
296
+ "learning_rate": 9.25e-05,
297
+ "loss": 4.0891,
298
+ "step": 1850
299
+ },
300
+ {
301
+ "epoch": 3.639081559435542,
302
+ "grad_norm": 25.684823989868164,
303
+ "learning_rate": 9.5e-05,
304
+ "loss": 4.0587,
305
+ "step": 1900
306
+ },
307
+ {
308
+ "epoch": 3.734752451566611,
309
+ "grad_norm": 24.86536407470703,
310
+ "learning_rate": 9.750000000000001e-05,
311
+ "loss": 4.0672,
312
+ "step": 1950
313
+ },
314
+ {
315
+ "epoch": 3.83042334369768,
316
+ "grad_norm": 23.63920021057129,
317
+ "learning_rate": 0.0001,
318
+ "loss": 4.7268,
319
+ "step": 2000
320
+ },
321
+ {
322
+ "epoch": 3.926094235828749,
323
+ "grad_norm": 25.422239303588867,
324
+ "learning_rate": 0.0001025,
325
+ "loss": 5.369,
326
+ "step": 2050
327
+ },
328
+ {
329
+ "epoch": 3.9988041138483617,
330
+ "eval_accuracy": 0.0,
331
+ "eval_loss": 4.845930099487305,
332
+ "eval_normalizer": 93665.0,
333
+ "eval_runtime": 98.1242,
334
+ "eval_samples_per_second": 544.789,
335
+ "eval_steps_per_second": 1.07,
336
+ "step": 2088
337
+ },
338
+ {
339
+ "epoch": 4.0229610141114565,
340
+ "grad_norm": 28.550003051757812,
341
+ "learning_rate": 0.000105,
342
+ "loss": 5.131,
343
+ "step": 2100
344
+ },
345
+ {
346
+ "epoch": 4.118631906242526,
347
+ "grad_norm": 24.855363845825195,
348
+ "learning_rate": 0.0001075,
349
+ "loss": 4.85,
350
+ "step": 2150
351
+ },
352
+ {
353
+ "epoch": 4.214302798373595,
354
+ "grad_norm": 28.85999870300293,
355
+ "learning_rate": 0.00011,
356
+ "loss": 3.6755,
357
+ "step": 2200
358
+ },
359
+ {
360
+ "epoch": 4.309973690504664,
361
+ "grad_norm": 26.07246971130371,
362
+ "learning_rate": 0.00011250000000000001,
363
+ "loss": 3.0516,
364
+ "step": 2250
365
+ },
366
+ {
367
+ "epoch": 4.405644582635733,
368
+ "grad_norm": 66.89540100097656,
369
+ "learning_rate": 0.000115,
370
+ "loss": 4.28,
371
+ "step": 2300
372
+ },
373
+ {
374
+ "epoch": 4.501315474766802,
375
+ "grad_norm": 23.402463912963867,
376
+ "learning_rate": 0.0001175,
377
+ "loss": 3.8331,
378
+ "step": 2350
379
+ },
380
+ {
381
+ "epoch": 4.596986366897871,
382
+ "grad_norm": 24.565364837646484,
383
+ "learning_rate": 0.00012,
384
+ "loss": 3.8233,
385
+ "step": 2400
386
+ },
387
+ {
388
+ "epoch": 4.69265725902894,
389
+ "grad_norm": 23.60770034790039,
390
+ "learning_rate": 0.0001225,
391
+ "loss": 3.7483,
392
+ "step": 2450
393
+ },
394
+ {
395
+ "epoch": 4.78832815116001,
396
+ "grad_norm": 19.10536003112793,
397
+ "learning_rate": 0.000125,
398
+ "loss": 4.1922,
399
+ "step": 2500
400
+ },
401
+ {
402
+ "epoch": 4.8839990432910785,
403
+ "grad_norm": 23.9981746673584,
404
+ "learning_rate": 0.0001275,
405
+ "loss": 4.5636,
406
+ "step": 2550
407
+ },
408
+ {
409
+ "epoch": 4.979669935422148,
410
+ "grad_norm": 52.10109329223633,
411
+ "learning_rate": 0.00013000000000000002,
412
+ "loss": 5.1324,
413
+ "step": 2600
414
+ },
415
+ {
416
+ "epoch": 4.998804113848362,
417
+ "eval_accuracy": 0.0,
418
+ "eval_loss": 4.496237754821777,
419
+ "eval_normalizer": 94313.0,
420
+ "eval_runtime": 86.4844,
421
+ "eval_samples_per_second": 618.112,
422
+ "eval_steps_per_second": 1.214,
423
+ "step": 2610
424
+ },
425
+ {
426
+ "epoch": 5.076536713704855,
427
+ "grad_norm": 23.186723709106445,
428
+ "learning_rate": 0.00013250000000000002,
429
+ "loss": 4.5323,
430
+ "step": 2650
431
+ },
432
+ {
433
+ "epoch": 5.172207605835925,
434
+ "grad_norm": 28.158355712890625,
435
+ "learning_rate": 0.000135,
436
+ "loss": 4.2118,
437
+ "step": 2700
438
+ },
439
+ {
440
+ "epoch": 5.2678784979669935,
441
+ "grad_norm": 28.501588821411133,
442
+ "learning_rate": 0.0001375,
443
+ "loss": 2.9381,
444
+ "step": 2750
445
+ },
446
+ {
447
+ "epoch": 5.363549390098063,
448
+ "grad_norm": 18.10572052001953,
449
+ "learning_rate": 0.00014000000000000001,
450
+ "loss": 3.4088,
451
+ "step": 2800
452
+ },
453
+ {
454
+ "epoch": 5.459220282229132,
455
+ "grad_norm": 26.142528533935547,
456
+ "learning_rate": 0.0001425,
457
+ "loss": 3.8884,
458
+ "step": 2850
459
+ },
460
+ {
461
+ "epoch": 5.5548911743602005,
462
+ "grad_norm": 22.223371505737305,
463
+ "learning_rate": 0.000145,
464
+ "loss": 3.6124,
465
+ "step": 2900
466
+ },
467
+ {
468
+ "epoch": 5.65056206649127,
469
+ "grad_norm": 24.494014739990234,
470
+ "learning_rate": 0.0001475,
471
+ "loss": 3.6075,
472
+ "step": 2950
473
+ },
474
+ {
475
+ "epoch": 5.746232958622339,
476
+ "grad_norm": 21.019962310791016,
477
+ "learning_rate": 0.00015,
478
+ "loss": 3.6963,
479
+ "step": 3000
480
+ },
481
+ {
482
+ "epoch": 5.8419038507534085,
483
+ "grad_norm": 20.909692764282227,
484
+ "learning_rate": 0.0001525,
485
+ "loss": 4.1607,
486
+ "step": 3050
487
+ },
488
+ {
489
+ "epoch": 5.937574742884477,
490
+ "grad_norm": 27.56682586669922,
491
+ "learning_rate": 0.000155,
492
+ "loss": 4.8602,
493
+ "step": 3100
494
+ },
495
+ {
496
+ "epoch": 5.998804113848362,
497
+ "eval_accuracy": 0.0,
498
+ "eval_loss": 4.2728495597839355,
499
+ "eval_normalizer": 93901.0,
500
+ "eval_runtime": 81.5686,
501
+ "eval_samples_per_second": 655.363,
502
+ "eval_steps_per_second": 1.287,
503
+ "step": 3132
504
+ }
505
+ ],
506
+ "logging_steps": 50,
507
+ "max_steps": 5220,
508
+ "num_input_tokens_seen": 0,
509
+ "num_train_epochs": 10,
510
+ "save_steps": 500,
511
+ "stateful_callbacks": {
512
+ "TrainerControl": {
513
+ "args": {
514
+ "should_epoch_stop": false,
515
+ "should_evaluate": false,
516
+ "should_log": false,
517
+ "should_save": true,
518
+ "should_training_stop": false
519
+ },
520
+ "attributes": {}
521
+ }
522
+ },
523
+ "total_flos": 3.4721573871609446e+17,
524
+ "train_batch_size": 128,
525
+ "trial_name": null,
526
+ "trial_params": null
527
+ }
checkpoints/checkpoint-3132/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1dba3751aa7691a086bca0b3920013c2e5e1509ccf2a749a7547f9bd962cdd0d
3
+ size 5432