mgh6 commited on
Commit
0c60715
·
verified ·
1 Parent(s): 4bceb41

Training in progress, epoch 3, checkpoint

Browse files
last-checkpoint/.ipynb_checkpoints/trainer_state-checkpoint.json ADDED
@@ -0,0 +1,333 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.9972065305108946,
5
+ "eval_steps": 50,
6
+ "global_step": 1004,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.1986467192252778,
13
+ "grad_norm": 16.624483108520508,
14
+ "learning_rate": 9.800796812749005e-05,
15
+ "loss": 223.0731,
16
+ "step": 50
17
+ },
18
+ {
19
+ "epoch": 0.1986467192252778,
20
+ "eval_loss": 218.2069549560547,
21
+ "eval_runtime": 20.7815,
22
+ "eval_samples_per_second": 40.806,
23
+ "eval_steps_per_second": 20.403,
24
+ "step": 50
25
+ },
26
+ {
27
+ "epoch": 0.3972934384505556,
28
+ "grad_norm": 22.61164665222168,
29
+ "learning_rate": 9.601593625498009e-05,
30
+ "loss": 219.2387,
31
+ "step": 100
32
+ },
33
+ {
34
+ "epoch": 0.3972934384505556,
35
+ "eval_loss": 214.27130126953125,
36
+ "eval_runtime": 20.7861,
37
+ "eval_samples_per_second": 40.797,
38
+ "eval_steps_per_second": 20.398,
39
+ "step": 100
40
+ },
41
+ {
42
+ "epoch": 0.5959401576758334,
43
+ "grad_norm": 39.08198928833008,
44
+ "learning_rate": 9.402390438247013e-05,
45
+ "loss": 206.0933,
46
+ "step": 150
47
+ },
48
+ {
49
+ "epoch": 0.5959401576758334,
50
+ "eval_loss": 208.94505310058594,
51
+ "eval_runtime": 20.7825,
52
+ "eval_samples_per_second": 40.804,
53
+ "eval_steps_per_second": 20.402,
54
+ "step": 150
55
+ },
56
+ {
57
+ "epoch": 0.7945868769011112,
58
+ "grad_norm": 49.450714111328125,
59
+ "learning_rate": 9.203187250996016e-05,
60
+ "loss": 207.5961,
61
+ "step": 200
62
+ },
63
+ {
64
+ "epoch": 0.7945868769011112,
65
+ "eval_loss": 200.57440185546875,
66
+ "eval_runtime": 20.7763,
67
+ "eval_samples_per_second": 40.816,
68
+ "eval_steps_per_second": 20.408,
69
+ "step": 200
70
+ },
71
+ {
72
+ "epoch": 0.993233596126389,
73
+ "grad_norm": 88.86530303955078,
74
+ "learning_rate": 9.00398406374502e-05,
75
+ "loss": 197.4489,
76
+ "step": 250
77
+ },
78
+ {
79
+ "epoch": 0.993233596126389,
80
+ "eval_loss": 193.4808807373047,
81
+ "eval_runtime": 20.7884,
82
+ "eval_samples_per_second": 40.792,
83
+ "eval_steps_per_second": 20.396,
84
+ "step": 250
85
+ },
86
+ {
87
+ "epoch": 1.1946737848407722,
88
+ "grad_norm": 94.41268157958984,
89
+ "learning_rate": 8.804780876494024e-05,
90
+ "loss": 186.8238,
91
+ "step": 300
92
+ },
93
+ {
94
+ "epoch": 1.1946737848407722,
95
+ "eval_loss": 183.36685180664062,
96
+ "eval_runtime": 20.7922,
97
+ "eval_samples_per_second": 40.785,
98
+ "eval_steps_per_second": 20.392,
99
+ "step": 300
100
+ },
101
+ {
102
+ "epoch": 1.39332050406605,
103
+ "grad_norm": 115.2780532836914,
104
+ "learning_rate": 8.605577689243029e-05,
105
+ "loss": 176.8817,
106
+ "step": 350
107
+ },
108
+ {
109
+ "epoch": 1.39332050406605,
110
+ "eval_loss": 167.37599182128906,
111
+ "eval_runtime": 20.7875,
112
+ "eval_samples_per_second": 40.794,
113
+ "eval_steps_per_second": 20.397,
114
+ "step": 350
115
+ },
116
+ {
117
+ "epoch": 1.5919672232913278,
118
+ "grad_norm": 108.17979431152344,
119
+ "learning_rate": 8.406374501992032e-05,
120
+ "loss": 165.4761,
121
+ "step": 400
122
+ },
123
+ {
124
+ "epoch": 1.5919672232913278,
125
+ "eval_loss": 148.36529541015625,
126
+ "eval_runtime": 20.7823,
127
+ "eval_samples_per_second": 40.804,
128
+ "eval_steps_per_second": 20.402,
129
+ "step": 400
130
+ },
131
+ {
132
+ "epoch": 1.7906139425166057,
133
+ "grad_norm": 181.7048797607422,
134
+ "learning_rate": 8.207171314741037e-05,
135
+ "loss": 149.9219,
136
+ "step": 450
137
+ },
138
+ {
139
+ "epoch": 1.7906139425166057,
140
+ "eval_loss": 144.7928924560547,
141
+ "eval_runtime": 20.7791,
142
+ "eval_samples_per_second": 40.81,
143
+ "eval_steps_per_second": 20.405,
144
+ "step": 450
145
+ },
146
+ {
147
+ "epoch": 1.9892606617418833,
148
+ "grad_norm": 637.717529296875,
149
+ "learning_rate": 8.00796812749004e-05,
150
+ "loss": 127.9147,
151
+ "step": 500
152
+ },
153
+ {
154
+ "epoch": 1.9892606617418833,
155
+ "eval_loss": 113.41303253173828,
156
+ "eval_runtime": 20.7782,
157
+ "eval_samples_per_second": 40.812,
158
+ "eval_steps_per_second": 20.406,
159
+ "step": 500
160
+ },
161
+ {
162
+ "epoch": 2.190700850456267,
163
+ "grad_norm": 1505.125244140625,
164
+ "learning_rate": 7.808764940239044e-05,
165
+ "loss": 106.9943,
166
+ "step": 550
167
+ },
168
+ {
169
+ "epoch": 2.190700850456267,
170
+ "eval_loss": 105.97330474853516,
171
+ "eval_runtime": 20.7864,
172
+ "eval_samples_per_second": 40.796,
173
+ "eval_steps_per_second": 20.398,
174
+ "step": 550
175
+ },
176
+ {
177
+ "epoch": 2.3893475696815445,
178
+ "grad_norm": 95.92086791992188,
179
+ "learning_rate": 7.609561752988048e-05,
180
+ "loss": 88.9087,
181
+ "step": 600
182
+ },
183
+ {
184
+ "epoch": 2.3893475696815445,
185
+ "eval_loss": 75.11954498291016,
186
+ "eval_runtime": 20.7814,
187
+ "eval_samples_per_second": 40.806,
188
+ "eval_steps_per_second": 20.403,
189
+ "step": 600
190
+ },
191
+ {
192
+ "epoch": 2.587994288906822,
193
+ "grad_norm": 484.91204833984375,
194
+ "learning_rate": 7.410358565737052e-05,
195
+ "loss": 95.5976,
196
+ "step": 650
197
+ },
198
+ {
199
+ "epoch": 2.587994288906822,
200
+ "eval_loss": 86.23988342285156,
201
+ "eval_runtime": 20.7754,
202
+ "eval_samples_per_second": 40.817,
203
+ "eval_steps_per_second": 20.409,
204
+ "step": 650
205
+ },
206
+ {
207
+ "epoch": 2.7866410081321,
208
+ "grad_norm": 46.8211669921875,
209
+ "learning_rate": 7.211155378486057e-05,
210
+ "loss": 83.614,
211
+ "step": 700
212
+ },
213
+ {
214
+ "epoch": 2.7866410081321,
215
+ "eval_loss": 63.02427673339844,
216
+ "eval_runtime": 20.7817,
217
+ "eval_samples_per_second": 40.805,
218
+ "eval_steps_per_second": 20.403,
219
+ "step": 700
220
+ },
221
+ {
222
+ "epoch": 2.985287727357378,
223
+ "grad_norm": 122.56575775146484,
224
+ "learning_rate": 7.01195219123506e-05,
225
+ "loss": 78.3681,
226
+ "step": 750
227
+ },
228
+ {
229
+ "epoch": 2.985287727357378,
230
+ "eval_loss": 36.502681732177734,
231
+ "eval_runtime": 20.778,
232
+ "eval_samples_per_second": 40.812,
233
+ "eval_steps_per_second": 20.406,
234
+ "step": 750
235
+ },
236
+ {
237
+ "epoch": 3.186727916071761,
238
+ "grad_norm": 352.10821533203125,
239
+ "learning_rate": 6.812749003984064e-05,
240
+ "loss": 38.2802,
241
+ "step": 800
242
+ },
243
+ {
244
+ "epoch": 3.186727916071761,
245
+ "eval_loss": 35.01713562011719,
246
+ "eval_runtime": 20.7907,
247
+ "eval_samples_per_second": 40.788,
248
+ "eval_steps_per_second": 20.394,
249
+ "step": 800
250
+ },
251
+ {
252
+ "epoch": 3.385374635297039,
253
+ "grad_norm": 97.40475463867188,
254
+ "learning_rate": 6.613545816733068e-05,
255
+ "loss": 27.1773,
256
+ "step": 850
257
+ },
258
+ {
259
+ "epoch": 3.385374635297039,
260
+ "eval_loss": 22.674545288085938,
261
+ "eval_runtime": 20.7869,
262
+ "eval_samples_per_second": 40.795,
263
+ "eval_steps_per_second": 20.397,
264
+ "step": 850
265
+ },
266
+ {
267
+ "epoch": 3.5840213545223167,
268
+ "grad_norm": 106.43690490722656,
269
+ "learning_rate": 6.414342629482072e-05,
270
+ "loss": 26.6469,
271
+ "step": 900
272
+ },
273
+ {
274
+ "epoch": 3.5840213545223167,
275
+ "eval_loss": 10.793374061584473,
276
+ "eval_runtime": 20.7814,
277
+ "eval_samples_per_second": 40.806,
278
+ "eval_steps_per_second": 20.403,
279
+ "step": 900
280
+ },
281
+ {
282
+ "epoch": 3.7826680737475944,
283
+ "grad_norm": 121.86284637451172,
284
+ "learning_rate": 6.215139442231077e-05,
285
+ "loss": 4.4939,
286
+ "step": 950
287
+ },
288
+ {
289
+ "epoch": 3.7826680737475944,
290
+ "eval_loss": -3.3120968341827393,
291
+ "eval_runtime": 20.7762,
292
+ "eval_samples_per_second": 40.816,
293
+ "eval_steps_per_second": 20.408,
294
+ "step": 950
295
+ },
296
+ {
297
+ "epoch": 3.9813147929728725,
298
+ "grad_norm": 148.8233642578125,
299
+ "learning_rate": 6.01593625498008e-05,
300
+ "loss": 19.1452,
301
+ "step": 1000
302
+ },
303
+ {
304
+ "epoch": 3.9813147929728725,
305
+ "eval_loss": -23.70981788635254,
306
+ "eval_runtime": 20.7822,
307
+ "eval_samples_per_second": 40.804,
308
+ "eval_steps_per_second": 20.402,
309
+ "step": 1000
310
+ }
311
+ ],
312
+ "logging_steps": 50,
313
+ "max_steps": 2510,
314
+ "num_input_tokens_seen": 0,
315
+ "num_train_epochs": 10,
316
+ "save_steps": 500,
317
+ "stateful_callbacks": {
318
+ "TrainerControl": {
319
+ "args": {
320
+ "should_epoch_stop": false,
321
+ "should_evaluate": false,
322
+ "should_log": false,
323
+ "should_save": true,
324
+ "should_training_stop": false
325
+ },
326
+ "attributes": {}
327
+ }
328
+ },
329
+ "total_flos": 0.0,
330
+ "train_batch_size": 2,
331
+ "trial_name": null,
332
+ "trial_params": null
333
+ }
last-checkpoint/model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6fb1ac3363635a3b0090068f8794f7a3fff0ed98d9564aa75664f20978996f6a
3
  size 2610104820
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6549006c4278533d56c4a872630e58d454236c45e6b6f37240a67cc4a8d1149f
3
  size 2610104820
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:44cc8995b543f42db2626d1ca4565368b0ca96d47a236ece7b6e6f2efb38e27c
3
  size 5210004271
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7ef81cbae86f3da4764bec2f9b3bc85de611d64cc8367d35920927dc54f9266
3
  size 5210004271
last-checkpoint/rng_state.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b7964ea9b27b8ec2d253f8e37dd1b85f1110775e8fe1122f0a165ddc0fdee12d
3
  size 15006
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3dc11eb7ffb894aa9af0579bc9e38efeb9cdc6220ba95d6dbad8bccb8be011a
3
  size 15006
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:863cf4a7956086b9fd348589f422e8af87ea5d78b5c7d0e595c32a5aaf7e77e5
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fca60d3e211c90170c539f1ba6815cff92dba636daaf20fea2fac1d2459b3c9f
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 2.9972065305108946,
5
  "eval_steps": 50,
6
- "global_step": 753,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
@@ -232,6 +232,81 @@
232
  "eval_samples_per_second": 31.338,
233
  "eval_steps_per_second": 15.669,
234
  "step": 750
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  }
236
  ],
237
  "logging_steps": 50,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 3.9972065305108946,
5
  "eval_steps": 50,
6
+ "global_step": 1004,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
 
232
  "eval_samples_per_second": 31.338,
233
  "eval_steps_per_second": 15.669,
234
  "step": 750
235
+ },
236
+ {
237
+ "epoch": 3.186727916071761,
238
+ "grad_norm": 0.14813293516635895,
239
+ "learning_rate": 6.812749003984064e-05,
240
+ "loss": 0.0945,
241
+ "step": 800
242
+ },
243
+ {
244
+ "epoch": 3.186727916071761,
245
+ "eval_loss": 0.14082284271717072,
246
+ "eval_runtime": 27.0281,
247
+ "eval_samples_per_second": 31.375,
248
+ "eval_steps_per_second": 15.687,
249
+ "step": 800
250
+ },
251
+ {
252
+ "epoch": 3.385374635297039,
253
+ "grad_norm": 0.14571304619312286,
254
+ "learning_rate": 6.613545816733068e-05,
255
+ "loss": 0.085,
256
+ "step": 850
257
+ },
258
+ {
259
+ "epoch": 3.385374635297039,
260
+ "eval_loss": 0.13174280524253845,
261
+ "eval_runtime": 27.0267,
262
+ "eval_samples_per_second": 31.376,
263
+ "eval_steps_per_second": 15.688,
264
+ "step": 850
265
+ },
266
+ {
267
+ "epoch": 3.5840213545223167,
268
+ "grad_norm": 0.15489640831947327,
269
+ "learning_rate": 6.414342629482072e-05,
270
+ "loss": 0.0774,
271
+ "step": 900
272
+ },
273
+ {
274
+ "epoch": 3.5840213545223167,
275
+ "eval_loss": 0.12704746425151825,
276
+ "eval_runtime": 27.0289,
277
+ "eval_samples_per_second": 31.374,
278
+ "eval_steps_per_second": 15.687,
279
+ "step": 900
280
+ },
281
+ {
282
+ "epoch": 3.7826680737475944,
283
+ "grad_norm": 0.15014521777629852,
284
+ "learning_rate": 6.215139442231077e-05,
285
+ "loss": 0.0761,
286
+ "step": 950
287
+ },
288
+ {
289
+ "epoch": 3.7826680737475944,
290
+ "eval_loss": 0.12169376760721207,
291
+ "eval_runtime": 27.0479,
292
+ "eval_samples_per_second": 31.352,
293
+ "eval_steps_per_second": 15.676,
294
+ "step": 950
295
+ },
296
+ {
297
+ "epoch": 3.9813147929728725,
298
+ "grad_norm": 0.13176779448986053,
299
+ "learning_rate": 6.01593625498008e-05,
300
+ "loss": 0.071,
301
+ "step": 1000
302
+ },
303
+ {
304
+ "epoch": 3.9813147929728725,
305
+ "eval_loss": 0.1125979870557785,
306
+ "eval_runtime": 27.0251,
307
+ "eval_samples_per_second": 31.378,
308
+ "eval_steps_per_second": 15.689,
309
+ "step": 1000
310
  }
311
  ],
312
  "logging_steps": 50,