Hugging Face's logo

loris3
/
stratified_10m_curriculum_roberta_perplexity_increasing

Fill-Mask
Transformers
Safetensors
roberta
Model card Files
xet
Community
loris3 commited on
Commit
36c5a03
·
verified ·
1 Parent(s): 23f8593

Upload folder using huggingface_hub

Browse files
Files changed (7) hide show
  1. checkpoints/checkpoint-1566/config.json +26 -0
  2. checkpoints/checkpoint-1566/model.safetensors +3 -0
  3. checkpoints/checkpoint-1566/optimizer.pt +3 -0
  4. checkpoints/checkpoint-1566/rng_state.pth +3 -0
  5. checkpoints/checkpoint-1566/scheduler.pt +3 -0
  6. checkpoints/checkpoint-1566/trainer_state.json +280 -0
  7. checkpoints/checkpoint-1566/training_args.bin +3 -0
checkpoints/checkpoint-1566/config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "RobertaForMaskedLM"
4
+ ],
5
+ "attention_probs_dropout_prob": 0.1,
6
+ "bos_token_id": 0,
7
+ "classifier_dropout": null,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 768,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 3072,
14
+ "layer_norm_eps": 1e-05,
15
+ "max_position_embeddings": 514,
16
+ "model_type": "roberta",
17
+ "num_attention_heads": 12,
18
+ "num_hidden_layers": 12,
19
+ "pad_token_id": 1,
20
+ "position_embedding_type": "absolute",
21
+ "torch_dtype": "float32",
22
+ "transformers_version": "4.47.0",
23
+ "type_vocab_size": 1,
24
+ "use_cache": true,
25
+ "vocab_size": 52000
26
+ }
checkpoints/checkpoint-1566/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b6c96336244c220fad99766540ca323a0946c17db8f78eee8fea35cb9a2fc86
3
+ size 504150808
checkpoints/checkpoint-1566/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d164fb1dc5e96c2686b91e740aaffe5521874a421a13b8797f08003b31335b44
3
+ size 1008422138
checkpoints/checkpoint-1566/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb723cd18760eca8be936c34ff92c7bd2b84c26a701e7c3ef692ba2724757037
3
+ size 14244
checkpoints/checkpoint-1566/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:866bb51799197d30f6eadd2a8b6aa966ec99226830a32c4d2e1b0ea4093030c0
3
+ size 1064
checkpoints/checkpoint-1566/trainer_state.json ADDED
@@ -0,0 +1,280 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.9988041138483617,
5
+ "eval_steps": 500,
6
+ "global_step": 1566,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.09567089213106912,
13
+ "grad_norm": 117.45254516601562,
14
+ "learning_rate": 2.5e-06,
15
+ "loss": 31905.095,
16
+ "step": 50
17
+ },
18
+ {
19
+ "epoch": 0.19134178426213824,
20
+ "grad_norm": 71.91663360595703,
21
+ "learning_rate": 5e-06,
22
+ "loss": 2260638924.8,
23
+ "step": 100
24
+ },
25
+ {
26
+ "epoch": 0.2870126763932074,
27
+ "grad_norm": 56.18342590332031,
28
+ "learning_rate": 7.5e-06,
29
+ "loss": 63.7996,
30
+ "step": 150
31
+ },
32
+ {
33
+ "epoch": 0.3826835685242765,
34
+ "grad_norm": 47.14400100708008,
35
+ "learning_rate": 1e-05,
36
+ "loss": 12.4659,
37
+ "step": 200
38
+ },
39
+ {
40
+ "epoch": 0.4783544606553456,
41
+ "grad_norm": 57.82217025756836,
42
+ "learning_rate": 1.25e-05,
43
+ "loss": 9.1207,
44
+ "step": 250
45
+ },
46
+ {
47
+ "epoch": 0.5740253527864148,
48
+ "grad_norm": 36.61322021484375,
49
+ "learning_rate": 1.5e-05,
50
+ "loss": 8.5922,
51
+ "step": 300
52
+ },
53
+ {
54
+ "epoch": 0.6696962449174838,
55
+ "grad_norm": 30.304691314697266,
56
+ "learning_rate": 1.7500000000000002e-05,
57
+ "loss": 7.5559,
58
+ "step": 350
59
+ },
60
+ {
61
+ "epoch": 0.765367137048553,
62
+ "grad_norm": 24.35045051574707,
63
+ "learning_rate": 2e-05,
64
+ "loss": 7.2955,
65
+ "step": 400
66
+ },
67
+ {
68
+ "epoch": 0.8610380291796221,
69
+ "grad_norm": 24.114551544189453,
70
+ "learning_rate": 2.2499999999999998e-05,
71
+ "loss": 7.2298,
72
+ "step": 450
73
+ },
74
+ {
75
+ "epoch": 0.9567089213106912,
76
+ "grad_norm": 22.68138313293457,
77
+ "learning_rate": 2.5e-05,
78
+ "loss": 7.5636,
79
+ "step": 500
80
+ },
81
+ {
82
+ "epoch": 0.9988041138483617,
83
+ "eval_accuracy": 0.0,
84
+ "eval_loss": 10.570171356201172,
85
+ "eval_normalizer": 94012.0,
86
+ "eval_runtime": 76.7018,
87
+ "eval_samples_per_second": 696.945,
88
+ "eval_steps_per_second": 1.369,
89
+ "step": 522
90
+ },
91
+ {
92
+ "epoch": 1.0535756995933987,
93
+ "grad_norm": 136.1419219970703,
94
+ "learning_rate": 2.75e-05,
95
+ "loss": 244134.34,
96
+ "step": 550
97
+ },
98
+ {
99
+ "epoch": 1.1492465917244679,
100
+ "grad_norm": 83.12769317626953,
101
+ "learning_rate": 3e-05,
102
+ "loss": 422854923386.88,
103
+ "step": 600
104
+ },
105
+ {
106
+ "epoch": 1.244917483855537,
107
+ "grad_norm": 75.21563720703125,
108
+ "learning_rate": 3.2500000000000004e-05,
109
+ "loss": 122855.21,
110
+ "step": 650
111
+ },
112
+ {
113
+ "epoch": 1.3405883759866062,
114
+ "grad_norm": 53.24630355834961,
115
+ "learning_rate": 3.5000000000000004e-05,
116
+ "loss": 8.0066,
117
+ "step": 700
118
+ },
119
+ {
120
+ "epoch": 1.4362592681176751,
121
+ "grad_norm": 78.36145782470703,
122
+ "learning_rate": 3.75e-05,
123
+ "loss": 5.7671,
124
+ "step": 750
125
+ },
126
+ {
127
+ "epoch": 1.5319301602487443,
128
+ "grad_norm": 42.857173919677734,
129
+ "learning_rate": 4e-05,
130
+ "loss": 5.6876,
131
+ "step": 800
132
+ },
133
+ {
134
+ "epoch": 1.6276010523798135,
135
+ "grad_norm": 68.13167572021484,
136
+ "learning_rate": 4.25e-05,
137
+ "loss": 5.532,
138
+ "step": 850
139
+ },
140
+ {
141
+ "epoch": 1.7232719445108826,
142
+ "grad_norm": 84.04206085205078,
143
+ "learning_rate": 4.4999999999999996e-05,
144
+ "loss": 5.8722,
145
+ "step": 900
146
+ },
147
+ {
148
+ "epoch": 1.8189428366419516,
149
+ "grad_norm": 172.8482666015625,
150
+ "learning_rate": 4.75e-05,
151
+ "loss": 6.2774,
152
+ "step": 950
153
+ },
154
+ {
155
+ "epoch": 1.9146137287730207,
156
+ "grad_norm": 71.3455810546875,
157
+ "learning_rate": 5e-05,
158
+ "loss": 6.8607,
159
+ "step": 1000
160
+ },
161
+ {
162
+ "epoch": 1.9988041138483617,
163
+ "eval_accuracy": 0.0,
164
+ "eval_loss": 12.504220962524414,
165
+ "eval_normalizer": 93558.0,
166
+ "eval_runtime": 76.0703,
167
+ "eval_samples_per_second": 702.731,
168
+ "eval_steps_per_second": 1.38,
169
+ "step": 1044
170
+ },
171
+ {
172
+ "epoch": 2.0114805070557282,
173
+ "grad_norm": 0.0,
174
+ "learning_rate": 5.25e-05,
175
+ "loss": 49.308,
176
+ "step": 1050
177
+ },
178
+ {
179
+ "epoch": 2.1071513991867974,
180
+ "grad_norm": 95.2462158203125,
181
+ "learning_rate": 5.5e-05,
182
+ "loss": 15104857.6,
183
+ "step": 1100
184
+ },
185
+ {
186
+ "epoch": 2.2028222913178666,
187
+ "grad_norm": 47.490631103515625,
188
+ "learning_rate": 5.75e-05,
189
+ "loss": 164198594.56,
190
+ "step": 1150
191
+ },
192
+ {
193
+ "epoch": 2.2984931834489357,
194
+ "grad_norm": 36.559452056884766,
195
+ "learning_rate": 6e-05,
196
+ "loss": 19.3007,
197
+ "step": 1200
198
+ },
199
+ {
200
+ "epoch": 2.394164075580005,
201
+ "grad_norm": 71.66349029541016,
202
+ "learning_rate": 6.25e-05,
203
+ "loss": 4.3654,
204
+ "step": 1250
205
+ },
206
+ {
207
+ "epoch": 2.489834967711074,
208
+ "grad_norm": 62.64774703979492,
209
+ "learning_rate": 6.500000000000001e-05,
210
+ "loss": 4.5691,
211
+ "step": 1300
212
+ },
213
+ {
214
+ "epoch": 2.5855058598421428,
215
+ "grad_norm": 82.2427749633789,
216
+ "learning_rate": 6.75e-05,
217
+ "loss": 4.6713,
218
+ "step": 1350
219
+ },
220
+ {
221
+ "epoch": 2.6811767519732124,
222
+ "grad_norm": 28.598220825195312,
223
+ "learning_rate": 7.000000000000001e-05,
224
+ "loss": 4.9054,
225
+ "step": 1400
226
+ },
227
+ {
228
+ "epoch": 2.776847644104281,
229
+ "grad_norm": 55.590911865234375,
230
+ "learning_rate": 7.25e-05,
231
+ "loss": 5.3171,
232
+ "step": 1450
233
+ },
234
+ {
235
+ "epoch": 2.8725185362353503,
236
+ "grad_norm": 52.226806640625,
237
+ "learning_rate": 7.5e-05,
238
+ "loss": 5.8207,
239
+ "step": 1500
240
+ },
241
+ {
242
+ "epoch": 2.9681894283664194,
243
+ "grad_norm": 41.47724151611328,
244
+ "learning_rate": 7.75e-05,
245
+ "loss": 6.6854,
246
+ "step": 1550
247
+ },
248
+ {
249
+ "epoch": 2.9988041138483617,
250
+ "eval_accuracy": 0.0,
251
+ "eval_loss": 13.740432739257812,
252
+ "eval_normalizer": 94216.0,
253
+ "eval_runtime": 75.872,
254
+ "eval_samples_per_second": 704.568,
255
+ "eval_steps_per_second": 1.384,
256
+ "step": 1566
257
+ }
258
+ ],
259
+ "logging_steps": 50,
260
+ "max_steps": 5220,
261
+ "num_input_tokens_seen": 0,
262
+ "num_train_epochs": 10,
263
+ "save_steps": 500,
264
+ "stateful_callbacks": {
265
+ "TrainerControl": {
266
+ "args": {
267
+ "should_epoch_stop": false,
268
+ "should_evaluate": false,
269
+ "should_log": false,
270
+ "should_save": true,
271
+ "should_training_stop": false
272
+ },
273
+ "attributes": {}
274
+ }
275
+ },
276
+ "total_flos": 1.2563861448012595e+17,
277
+ "train_batch_size": 128,
278
+ "trial_name": null,
279
+ "trial_params": null
280
+ }
checkpoints/checkpoint-1566/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b90d3ba093dd4889703652a506228505b353e93a789ce9359f2962066d1f58a8
3
+ size 5432