oribrand commited on
Commit
5466ad4
·
verified ·
1 Parent(s): b02f402

Upload 7 files

Browse files
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "openai-community/gpt2-large",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPT2LMHeadModel"
6
+ ],
7
+ "attn_pdrop": 0.1,
8
+ "bos_token_id": 50256,
9
+ "embd_pdrop": 0.1,
10
+ "eos_token_id": 50256,
11
+ "initializer_range": 0.02,
12
+ "layer_norm_epsilon": 1e-05,
13
+ "model_type": "gpt2",
14
+ "n_ctx": 1024,
15
+ "n_embd": 1280,
16
+ "n_head": 20,
17
+ "n_inner": null,
18
+ "n_layer": 36,
19
+ "n_positions": 1024,
20
+ "reorder_and_upcast_attn": false,
21
+ "resid_pdrop": 0.1,
22
+ "scale_attn_by_inverse_layer_idx": false,
23
+ "scale_attn_weights": true,
24
+ "summary_activation": null,
25
+ "summary_first_dropout": 0.1,
26
+ "summary_proj_to_labels": true,
27
+ "summary_type": "cls_index",
28
+ "summary_use_proj": true,
29
+ "task_specific_params": {
30
+ "text-generation": {
31
+ "do_sample": true,
32
+ "max_length": 50
33
+ }
34
+ },
35
+ "torch_dtype": "float32",
36
+ "transformers_version": "4.38.2",
37
+ "use_cache": true,
38
+ "vocab_size": 50257
39
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "eos_token_id": 50256,
5
+ "transformers_version": "4.38.2"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51df8d8fa291fd942caadb6325eff449e32976741d296f02aa26f9211e221514
3
+ size 3096165928
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"model_max_length": 1024}
trainer_state.json ADDED
@@ -0,0 +1,315 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 2.097651720046997,
3
+ "best_model_checkpoint": "/content/models/openai-community-gpt2-large/checkpoint-1460",
4
+ "epoch": 5.0,
5
+ "eval_steps": 365,
6
+ "global_step": 7290,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.25,
13
+ "grad_norm": 4.67961311340332,
14
+ "learning_rate": 4.7496570644718796e-05,
15
+ "loss": 2.3306,
16
+ "step": 365
17
+ },
18
+ {
19
+ "epoch": 0.25,
20
+ "eval_loss": 2.2394158840179443,
21
+ "eval_runtime": 3.8004,
22
+ "eval_samples_per_second": 53.942,
23
+ "eval_steps_per_second": 6.841,
24
+ "step": 365
25
+ },
26
+ {
27
+ "epoch": 0.5,
28
+ "grad_norm": 3.010347366333008,
29
+ "learning_rate": 4.499314128943759e-05,
30
+ "loss": 2.1698,
31
+ "step": 730
32
+ },
33
+ {
34
+ "epoch": 0.5,
35
+ "eval_loss": 2.153510332107544,
36
+ "eval_runtime": 3.8111,
37
+ "eval_samples_per_second": 53.79,
38
+ "eval_steps_per_second": 6.822,
39
+ "step": 730
40
+ },
41
+ {
42
+ "epoch": 0.75,
43
+ "grad_norm": 2.377396583557129,
44
+ "learning_rate": 4.248971193415638e-05,
45
+ "loss": 2.1126,
46
+ "step": 1095
47
+ },
48
+ {
49
+ "epoch": 0.75,
50
+ "eval_loss": 2.1220626831054688,
51
+ "eval_runtime": 3.8077,
52
+ "eval_samples_per_second": 53.839,
53
+ "eval_steps_per_second": 6.828,
54
+ "step": 1095
55
+ },
56
+ {
57
+ "epoch": 1.0,
58
+ "grad_norm": 2.0250906944274902,
59
+ "learning_rate": 3.9986282578875175e-05,
60
+ "loss": 2.0649,
61
+ "step": 1460
62
+ },
63
+ {
64
+ "epoch": 1.0,
65
+ "eval_loss": 2.097651720046997,
66
+ "eval_runtime": 3.8144,
67
+ "eval_samples_per_second": 53.744,
68
+ "eval_steps_per_second": 6.816,
69
+ "step": 1460
70
+ },
71
+ {
72
+ "epoch": 1.25,
73
+ "grad_norm": 3.449071168899536,
74
+ "learning_rate": 3.748285322359397e-05,
75
+ "loss": 1.6887,
76
+ "step": 1825
77
+ },
78
+ {
79
+ "epoch": 1.25,
80
+ "eval_loss": 2.125108003616333,
81
+ "eval_runtime": 3.8167,
82
+ "eval_samples_per_second": 53.712,
83
+ "eval_steps_per_second": 6.812,
84
+ "step": 1825
85
+ },
86
+ {
87
+ "epoch": 1.5,
88
+ "grad_norm": 2.925269365310669,
89
+ "learning_rate": 3.497942386831276e-05,
90
+ "loss": 1.7126,
91
+ "step": 2190
92
+ },
93
+ {
94
+ "epoch": 1.5,
95
+ "eval_loss": 2.1161348819732666,
96
+ "eval_runtime": 3.828,
97
+ "eval_samples_per_second": 53.553,
98
+ "eval_steps_per_second": 6.792,
99
+ "step": 2190
100
+ },
101
+ {
102
+ "epoch": 1.75,
103
+ "grad_norm": 3.348761796951294,
104
+ "learning_rate": 3.247599451303155e-05,
105
+ "loss": 1.7253,
106
+ "step": 2555
107
+ },
108
+ {
109
+ "epoch": 1.75,
110
+ "eval_loss": 2.1163294315338135,
111
+ "eval_runtime": 3.8247,
112
+ "eval_samples_per_second": 53.599,
113
+ "eval_steps_per_second": 6.798,
114
+ "step": 2555
115
+ },
116
+ {
117
+ "epoch": 2.0,
118
+ "grad_norm": 2.501504898071289,
119
+ "learning_rate": 2.9972565157750345e-05,
120
+ "loss": 1.6857,
121
+ "step": 2920
122
+ },
123
+ {
124
+ "epoch": 2.0,
125
+ "eval_loss": 2.1025500297546387,
126
+ "eval_runtime": 3.8312,
127
+ "eval_samples_per_second": 53.508,
128
+ "eval_steps_per_second": 6.786,
129
+ "step": 2920
130
+ },
131
+ {
132
+ "epoch": 2.25,
133
+ "grad_norm": 4.346751689910889,
134
+ "learning_rate": 2.7469135802469138e-05,
135
+ "loss": 1.1799,
136
+ "step": 3285
137
+ },
138
+ {
139
+ "epoch": 2.25,
140
+ "eval_loss": 2.34588360786438,
141
+ "eval_runtime": 3.8313,
142
+ "eval_samples_per_second": 53.507,
143
+ "eval_steps_per_second": 6.786,
144
+ "step": 3285
145
+ },
146
+ {
147
+ "epoch": 2.5,
148
+ "grad_norm": 5.11303186416626,
149
+ "learning_rate": 2.496570644718793e-05,
150
+ "loss": 1.1877,
151
+ "step": 3650
152
+ },
153
+ {
154
+ "epoch": 2.5,
155
+ "eval_loss": 2.343195676803589,
156
+ "eval_runtime": 3.8294,
157
+ "eval_samples_per_second": 53.534,
158
+ "eval_steps_per_second": 6.79,
159
+ "step": 3650
160
+ },
161
+ {
162
+ "epoch": 2.75,
163
+ "grad_norm": 4.008133411407471,
164
+ "learning_rate": 2.246227709190672e-05,
165
+ "loss": 1.1877,
166
+ "step": 4015
167
+ },
168
+ {
169
+ "epoch": 2.75,
170
+ "eval_loss": 2.378342390060425,
171
+ "eval_runtime": 3.8264,
172
+ "eval_samples_per_second": 53.574,
173
+ "eval_steps_per_second": 6.795,
174
+ "step": 4015
175
+ },
176
+ {
177
+ "epoch": 3.0,
178
+ "grad_norm": 3.2181403636932373,
179
+ "learning_rate": 1.9958847736625517e-05,
180
+ "loss": 1.1932,
181
+ "step": 4380
182
+ },
183
+ {
184
+ "epoch": 3.0,
185
+ "eval_loss": 2.3641903400421143,
186
+ "eval_runtime": 3.8349,
187
+ "eval_samples_per_second": 53.457,
188
+ "eval_steps_per_second": 6.78,
189
+ "step": 4380
190
+ },
191
+ {
192
+ "epoch": 3.25,
193
+ "grad_norm": 5.110302448272705,
194
+ "learning_rate": 1.7455418381344307e-05,
195
+ "loss": 0.7276,
196
+ "step": 4745
197
+ },
198
+ {
199
+ "epoch": 3.25,
200
+ "eval_loss": 2.6530215740203857,
201
+ "eval_runtime": 3.834,
202
+ "eval_samples_per_second": 53.469,
203
+ "eval_steps_per_second": 6.781,
204
+ "step": 4745
205
+ },
206
+ {
207
+ "epoch": 3.5,
208
+ "grad_norm": 4.757246971130371,
209
+ "learning_rate": 1.49519890260631e-05,
210
+ "loss": 0.7138,
211
+ "step": 5110
212
+ },
213
+ {
214
+ "epoch": 3.5,
215
+ "eval_loss": 2.721984386444092,
216
+ "eval_runtime": 3.8333,
217
+ "eval_samples_per_second": 53.479,
218
+ "eval_steps_per_second": 6.783,
219
+ "step": 5110
220
+ },
221
+ {
222
+ "epoch": 3.76,
223
+ "grad_norm": 3.532853603363037,
224
+ "learning_rate": 1.2448559670781894e-05,
225
+ "loss": 0.7135,
226
+ "step": 5475
227
+ },
228
+ {
229
+ "epoch": 3.76,
230
+ "eval_loss": 2.6952061653137207,
231
+ "eval_runtime": 3.8325,
232
+ "eval_samples_per_second": 53.489,
233
+ "eval_steps_per_second": 6.784,
234
+ "step": 5475
235
+ },
236
+ {
237
+ "epoch": 4.01,
238
+ "grad_norm": 3.3618454933166504,
239
+ "learning_rate": 9.945130315500687e-06,
240
+ "loss": 0.6983,
241
+ "step": 5840
242
+ },
243
+ {
244
+ "epoch": 4.01,
245
+ "eval_loss": 2.7304930686950684,
246
+ "eval_runtime": 3.8317,
247
+ "eval_samples_per_second": 53.502,
248
+ "eval_steps_per_second": 6.786,
249
+ "step": 5840
250
+ },
251
+ {
252
+ "epoch": 4.26,
253
+ "grad_norm": 4.5132622718811035,
254
+ "learning_rate": 7.441700960219479e-06,
255
+ "loss": 0.4195,
256
+ "step": 6205
257
+ },
258
+ {
259
+ "epoch": 4.26,
260
+ "eval_loss": 2.9191341400146484,
261
+ "eval_runtime": 3.8313,
262
+ "eval_samples_per_second": 53.506,
263
+ "eval_steps_per_second": 6.786,
264
+ "step": 6205
265
+ },
266
+ {
267
+ "epoch": 4.51,
268
+ "grad_norm": 5.9186201095581055,
269
+ "learning_rate": 4.938271604938272e-06,
270
+ "loss": 0.4193,
271
+ "step": 6570
272
+ },
273
+ {
274
+ "epoch": 4.51,
275
+ "eval_loss": 2.9266867637634277,
276
+ "eval_runtime": 3.8312,
277
+ "eval_samples_per_second": 53.509,
278
+ "eval_steps_per_second": 6.786,
279
+ "step": 6570
280
+ },
281
+ {
282
+ "epoch": 4.76,
283
+ "grad_norm": 5.357140064239502,
284
+ "learning_rate": 2.4348422496570644e-06,
285
+ "loss": 0.4239,
286
+ "step": 6935
287
+ },
288
+ {
289
+ "epoch": 4.76,
290
+ "eval_loss": 2.922820568084717,
291
+ "eval_runtime": 3.8273,
292
+ "eval_samples_per_second": 53.563,
293
+ "eval_steps_per_second": 6.793,
294
+ "step": 6935
295
+ },
296
+ {
297
+ "epoch": 5.0,
298
+ "step": 7290,
299
+ "total_flos": 3966079500288000.0,
300
+ "train_loss": 1.2395105746547872,
301
+ "train_runtime": 1985.2268,
302
+ "train_samples_per_second": 3.672,
303
+ "train_steps_per_second": 3.672
304
+ }
305
+ ],
306
+ "logging_steps": 365,
307
+ "max_steps": 7290,
308
+ "num_input_tokens_seen": 0,
309
+ "num_train_epochs": 5,
310
+ "save_steps": 365,
311
+ "total_flos": 3966079500288000.0,
312
+ "train_batch_size": 1,
313
+ "trial_name": null,
314
+ "trial_params": null
315
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f23952b3f9adeb554ee9b78753b42461f0bf14fc96eb354498d826d0a97d3fd9
3
+ size 4856