Hugging Face's logo

LocalLite
/
python-code-testgen-ministral-8B

Transformers
Safetensors
Model card Files
xet
 Community
djinn-anthrope commited on
Commit
e0dc2db
·
verified ·
1 Parent(s): 11373ee

Upload folder using huggingface_hub

Browse files
Files changed (20) hide show
  1. .gitattributes +4 -0
  2. adapter_config.json +42 -0
  3. checkpoint-400/adapter_config.json +42 -0
  4. checkpoint-400/special_tokens_map.json +24 -0
  5. checkpoint-400/tokenizer.json +3 -0
  6. checkpoint-400/tokenizer_config.json +0 -0
  7. checkpoint-400/trainer_state.json +522 -0
  8. checkpoint-500/adapter_config.json +42 -0
  9. checkpoint-500/special_tokens_map.json +24 -0
  10. checkpoint-500/tokenizer.json +3 -0
  11. checkpoint-500/tokenizer_config.json +0 -0
  12. checkpoint-500/trainer_state.json +644 -0
  13. checkpoint-600/adapter_config.json +42 -0
  14. checkpoint-600/special_tokens_map.json +24 -0
  15. checkpoint-600/tokenizer.json +3 -0
  16. checkpoint-600/tokenizer_config.json +0 -0
  17. checkpoint-600/trainer_state.json +766 -0
  18. special_tokens_map.json +24 -0
  19. tokenizer.json +3 -0
  20. tokenizer_config.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ checkpoint-400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
+ checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
+ checkpoint-600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
39
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/home/support/llm/Ministral-8B-Instruct-2410/",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 64,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "gate_proj",
29
+ "q_proj",
30
+ "k_proj",
31
+ "v_proj",
32
+ "o_proj",
33
+ "down_proj",
34
+ "up_proj"
35
+ ],
36
+ "target_parameters": null,
37
+ "task_type": "CAUSAL_LM",
38
+ "trainable_token_indices": null,
39
+ "use_dora": false,
40
+ "use_qalora": false,
41
+ "use_rslora": false
42
+ }
checkpoint-400/adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/home/support/llm/Ministral-8B-Instruct-2410/",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 64,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "gate_proj",
29
+ "q_proj",
30
+ "k_proj",
31
+ "v_proj",
32
+ "o_proj",
33
+ "down_proj",
34
+ "up_proj"
35
+ ],
36
+ "target_parameters": null,
37
+ "task_type": "CAUSAL_LM",
38
+ "trainable_token_indices": null,
39
+ "use_dora": false,
40
+ "use_qalora": false,
41
+ "use_rslora": false
42
+ }
checkpoint-400/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-400/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7edbeaf20dd7f571b5dd1c54d9ace4f9b6299127cc7ba2afb14a6d51a4a79a4
3
+ size 17078136
checkpoint-400/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-400/trainer_state.json ADDED
@@ -0,0 +1,522 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 400,
3
+ "best_metric": 0.43400639295578003,
4
+ "best_model_checkpoint": "./python-code-testgen-ministral-8B/checkpoint-400",
5
+ "epoch": 3.4814004376367613,
6
+ "eval_steps": 50,
7
+ "global_step": 400,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2537571787834167,
14
+ "epoch": 0.087527352297593,
15
+ "grad_norm": 0.5736712217330933,
16
+ "learning_rate": 5e-06,
17
+ "loss": 1.8641,
18
+ "mean_token_accuracy": 0.6149017184972763,
19
+ "num_tokens": 17392.0,
20
+ "step": 10
21
+ },
22
+ {
23
+ "entropy": 1.3263455376029014,
24
+ "epoch": 0.175054704595186,
25
+ "grad_norm": 0.7149288058280945,
26
+ "learning_rate": 9.999927156177032e-06,
27
+ "loss": 1.9444,
28
+ "mean_token_accuracy": 0.601305079460144,
29
+ "num_tokens": 33323.0,
30
+ "step": 20
31
+ },
32
+ {
33
+ "entropy": 1.3946671187877655,
34
+ "epoch": 0.26258205689277897,
35
+ "grad_norm": 0.6501050591468811,
36
+ "learning_rate": 9.991188465340766e-06,
37
+ "loss": 1.7875,
38
+ "mean_token_accuracy": 0.6164133206009865,
39
+ "num_tokens": 49246.0,
40
+ "step": 30
41
+ },
42
+ {
43
+ "entropy": 1.3958179399371147,
44
+ "epoch": 0.350109409190372,
45
+ "grad_norm": 0.5064170360565186,
46
+ "learning_rate": 9.96791018015489e-06,
47
+ "loss": 1.4612,
48
+ "mean_token_accuracy": 0.6549548670649529,
49
+ "num_tokens": 66175.0,
50
+ "step": 40
51
+ },
52
+ {
53
+ "entropy": 1.1895733758807183,
54
+ "epoch": 0.437636761487965,
55
+ "grad_norm": 0.7921415567398071,
56
+ "learning_rate": 9.930160111487717e-06,
57
+ "loss": 1.1449,
58
+ "mean_token_accuracy": 0.7270906791090965,
59
+ "num_tokens": 82566.0,
60
+ "step": 50
61
+ },
62
+ {
63
+ "epoch": 0.437636761487965,
64
+ "eval_entropy": 1.0364094030709914,
65
+ "eval_loss": 1.0214754343032837,
66
+ "eval_mean_token_accuracy": 0.7765023590605936,
67
+ "eval_num_tokens": 82566.0,
68
+ "eval_runtime": 10.3255,
69
+ "eval_samples_per_second": 7.845,
70
+ "eval_steps_per_second": 7.845,
71
+ "step": 50
72
+ },
73
+ {
74
+ "entropy": 0.8925203159451485,
75
+ "epoch": 0.5251641137855579,
76
+ "grad_norm": 0.34061136841773987,
77
+ "learning_rate": 9.87804822727352e-06,
78
+ "loss": 0.9039,
79
+ "mean_token_accuracy": 0.817305064201355,
80
+ "num_tokens": 99237.0,
81
+ "step": 60
82
+ },
83
+ {
84
+ "entropy": 0.6578707128763199,
85
+ "epoch": 0.612691466083151,
86
+ "grad_norm": 0.3180134892463684,
87
+ "learning_rate": 9.811726332170153e-06,
88
+ "loss": 0.7,
89
+ "mean_token_accuracy": 0.8500439867377281,
90
+ "num_tokens": 117524.0,
91
+ "step": 70
92
+ },
93
+ {
94
+ "entropy": 0.5739984951913357,
95
+ "epoch": 0.700218818380744,
96
+ "grad_norm": 0.4566318094730377,
97
+ "learning_rate": 9.731387625344104e-06,
98
+ "loss": 0.6788,
99
+ "mean_token_accuracy": 0.8602568313479424,
100
+ "num_tokens": 135866.0,
101
+ "step": 80
102
+ },
103
+ {
104
+ "entropy": 0.5298320218920708,
105
+ "epoch": 0.787746170678337,
106
+ "grad_norm": 0.31226617097854614,
107
+ "learning_rate": 9.637266137671177e-06,
108
+ "loss": 0.5391,
109
+ "mean_token_accuracy": 0.8844943925738334,
110
+ "num_tokens": 152266.0,
111
+ "step": 90
112
+ },
113
+ {
114
+ "entropy": 0.5451121114194393,
115
+ "epoch": 0.87527352297593,
116
+ "grad_norm": 0.20949594676494598,
117
+ "learning_rate": 9.529636049992235e-06,
118
+ "loss": 0.5172,
119
+ "mean_token_accuracy": 0.8867821574211121,
120
+ "num_tokens": 168374.0,
121
+ "step": 100
122
+ },
123
+ {
124
+ "epoch": 0.87527352297593,
125
+ "eval_entropy": 0.6073500057797373,
126
+ "eval_loss": 0.5493269562721252,
127
+ "eval_mean_token_accuracy": 0.8748943202289534,
128
+ "eval_num_tokens": 168374.0,
129
+ "eval_runtime": 10.3652,
130
+ "eval_samples_per_second": 7.815,
131
+ "eval_steps_per_second": 7.815,
132
+ "step": 100
133
+ },
134
+ {
135
+ "entropy": 0.5541586525738239,
136
+ "epoch": 0.962800875273523,
137
+ "grad_norm": 0.24901992082595825,
138
+ "learning_rate": 9.408810894410008e-06,
139
+ "loss": 0.5106,
140
+ "mean_token_accuracy": 0.8848323300480843,
141
+ "num_tokens": 185723.0,
142
+ "step": 110
143
+ },
144
+ {
145
+ "entropy": 0.5490001190353084,
146
+ "epoch": 1.0437636761487965,
147
+ "grad_norm": 0.2396228015422821,
148
+ "learning_rate": 9.275142640953599e-06,
149
+ "loss": 0.5232,
150
+ "mean_token_accuracy": 0.8872134911047446,
151
+ "num_tokens": 201213.0,
152
+ "step": 120
153
+ },
154
+ {
155
+ "entropy": 0.5084450632333756,
156
+ "epoch": 1.1312910284463895,
157
+ "grad_norm": 0.3080507218837738,
158
+ "learning_rate": 9.129020672271283e-06,
159
+ "loss": 0.464,
160
+ "mean_token_accuracy": 0.8925739243626595,
161
+ "num_tokens": 216879.0,
162
+ "step": 130
163
+ },
164
+ {
165
+ "entropy": 0.47396973669528963,
166
+ "epoch": 1.2188183807439825,
167
+ "grad_norm": 0.2524544894695282,
168
+ "learning_rate": 8.970870649338388e-06,
169
+ "loss": 0.442,
170
+ "mean_token_accuracy": 0.8983234956860542,
171
+ "num_tokens": 233348.0,
172
+ "step": 140
173
+ },
174
+ {
175
+ "entropy": 0.4726570636034012,
176
+ "epoch": 1.3063457330415755,
177
+ "grad_norm": 0.3230877220630646,
178
+ "learning_rate": 8.801153271484503e-06,
179
+ "loss": 0.4538,
180
+ "mean_token_accuracy": 0.8902932360768319,
181
+ "num_tokens": 250578.0,
182
+ "step": 150
183
+ },
184
+ {
185
+ "epoch": 1.3063457330415755,
186
+ "eval_entropy": 0.5106582998493572,
187
+ "eval_loss": 0.4790702760219574,
188
+ "eval_mean_token_accuracy": 0.8854184364095147,
189
+ "eval_num_tokens": 250578.0,
190
+ "eval_runtime": 10.3528,
191
+ "eval_samples_per_second": 7.824,
192
+ "eval_steps_per_second": 7.824,
193
+ "step": 150
194
+ },
195
+ {
196
+ "entropy": 0.48775711581110953,
197
+ "epoch": 1.3938730853391685,
198
+ "grad_norm": 0.27437666058540344,
199
+ "learning_rate": 8.620362934352109e-06,
200
+ "loss": 0.4712,
201
+ "mean_token_accuracy": 0.8893376395106316,
202
+ "num_tokens": 266848.0,
203
+ "step": 160
204
+ },
205
+ {
206
+ "entropy": 0.4436598148196936,
207
+ "epoch": 1.4814004376367615,
208
+ "grad_norm": 0.32482877373695374,
209
+ "learning_rate": 8.429026289696091e-06,
210
+ "loss": 0.4233,
211
+ "mean_token_accuracy": 0.8987479954957962,
212
+ "num_tokens": 282930.0,
213
+ "step": 170
214
+ },
215
+ {
216
+ "entropy": 0.44891218170523645,
217
+ "epoch": 1.5689277899343543,
218
+ "grad_norm": 0.3758986294269562,
219
+ "learning_rate": 8.227700711219492e-06,
220
+ "loss": 0.4315,
221
+ "mean_token_accuracy": 0.8931379184126854,
222
+ "num_tokens": 300722.0,
223
+ "step": 180
224
+ },
225
+ {
226
+ "entropy": 0.5109403282403946,
227
+ "epoch": 1.6564551422319473,
228
+ "grad_norm": 0.3557584583759308,
229
+ "learning_rate": 8.016972670914624e-06,
230
+ "loss": 0.5022,
231
+ "mean_token_accuracy": 0.8783736646175384,
232
+ "num_tokens": 317955.0,
233
+ "step": 190
234
+ },
235
+ {
236
+ "entropy": 0.46201174408197404,
237
+ "epoch": 1.7439824945295404,
238
+ "grad_norm": 0.3397444784641266,
239
+ "learning_rate": 7.797456030639314e-06,
240
+ "loss": 0.4591,
241
+ "mean_token_accuracy": 0.8974948301911354,
242
+ "num_tokens": 334732.0,
243
+ "step": 200
244
+ },
245
+ {
246
+ "epoch": 1.7439824945295404,
247
+ "eval_entropy": 0.47593313657943115,
248
+ "eval_loss": 0.45035651326179504,
249
+ "eval_mean_token_accuracy": 0.8885414180932222,
250
+ "eval_num_tokens": 334732.0,
251
+ "eval_runtime": 10.3648,
252
+ "eval_samples_per_second": 7.815,
253
+ "eval_steps_per_second": 7.815,
254
+ "step": 200
255
+ },
256
+ {
257
+ "entropy": 0.4485505182296038,
258
+ "epoch": 1.8315098468271334,
259
+ "grad_norm": 0.27542436122894287,
260
+ "learning_rate": 7.569790253905059e-06,
261
+ "loss": 0.4558,
262
+ "mean_token_accuracy": 0.8910033285617829,
263
+ "num_tokens": 353384.0,
264
+ "step": 210
265
+ },
266
+ {
267
+ "entropy": 0.450789924710989,
268
+ "epoch": 1.9190371991247264,
269
+ "grad_norm": 0.2546970546245575,
270
+ "learning_rate": 7.334638543086203e-06,
271
+ "loss": 0.4503,
272
+ "mean_token_accuracy": 0.8930477350950241,
273
+ "num_tokens": 370072.0,
274
+ "step": 220
275
+ },
276
+ {
277
+ "entropy": 0.45568384431503917,
278
+ "epoch": 2.0,
279
+ "grad_norm": 0.5460340976715088,
280
+ "learning_rate": 7.092685907476558e-06,
281
+ "loss": 0.4571,
282
+ "mean_token_accuracy": 0.898327117030685,
283
+ "num_tokens": 384804.0,
284
+ "step": 230
285
+ },
286
+ {
287
+ "entropy": 0.44126378260552884,
288
+ "epoch": 2.087527352297593,
289
+ "grad_norm": 0.3868609666824341,
290
+ "learning_rate": 6.844637167821327e-06,
291
+ "loss": 0.423,
292
+ "mean_token_accuracy": 0.8965987235307693,
293
+ "num_tokens": 401856.0,
294
+ "step": 240
295
+ },
296
+ {
297
+ "entropy": 0.42570732310414316,
298
+ "epoch": 2.175054704595186,
299
+ "grad_norm": 0.31367599964141846,
300
+ "learning_rate": 6.591214903137221e-06,
301
+ "loss": 0.4127,
302
+ "mean_token_accuracy": 0.8982817739248276,
303
+ "num_tokens": 418683.0,
304
+ "step": 250
305
+ },
306
+ {
307
+ "epoch": 2.175054704595186,
308
+ "eval_entropy": 0.45187407492855447,
309
+ "eval_loss": 0.44380050897598267,
310
+ "eval_mean_token_accuracy": 0.8904590201966557,
311
+ "eval_num_tokens": 418683.0,
312
+ "eval_runtime": 10.3129,
313
+ "eval_samples_per_second": 7.854,
314
+ "eval_steps_per_second": 7.854,
315
+ "step": 250
316
+ },
317
+ {
318
+ "entropy": 0.4225453745573759,
319
+ "epoch": 2.262582056892779,
320
+ "grad_norm": 0.3352673351764679,
321
+ "learning_rate": 6.333157345801808e-06,
322
+ "loss": 0.4236,
323
+ "mean_token_accuracy": 0.8970558911561965,
324
+ "num_tokens": 435678.0,
325
+ "step": 260
326
+ },
327
+ {
328
+ "entropy": 0.39916136525571344,
329
+ "epoch": 2.350109409190372,
330
+ "grad_norm": 0.3182181119918823,
331
+ "learning_rate": 6.0712162310438e-06,
332
+ "loss": 0.3953,
333
+ "mean_token_accuracy": 0.90475542396307,
334
+ "num_tokens": 453095.0,
335
+ "step": 270
336
+ },
337
+ {
338
+ "entropy": 0.46247223690152167,
339
+ "epoch": 2.437636761487965,
340
+ "grad_norm": 0.27779728174209595,
341
+ "learning_rate": 5.806154607098799e-06,
342
+ "loss": 0.4554,
343
+ "mean_token_accuracy": 0.8864100977778435,
344
+ "num_tokens": 470622.0,
345
+ "step": 280
346
+ },
347
+ {
348
+ "entropy": 0.4407076742500067,
349
+ "epoch": 2.525164113785558,
350
+ "grad_norm": 0.31952330470085144,
351
+ "learning_rate": 5.538744612409701e-06,
352
+ "loss": 0.4446,
353
+ "mean_token_accuracy": 0.8954811543226242,
354
+ "num_tokens": 489165.0,
355
+ "step": 290
356
+ },
357
+ {
358
+ "entropy": 0.4149217315018177,
359
+ "epoch": 2.612691466083151,
360
+ "grad_norm": 0.29808804392814636,
361
+ "learning_rate": 5.269765226346812e-06,
362
+ "loss": 0.4112,
363
+ "mean_token_accuracy": 0.9011785760521889,
364
+ "num_tokens": 504739.0,
365
+ "step": 300
366
+ },
367
+ {
368
+ "epoch": 2.612691466083151,
369
+ "eval_entropy": 0.4445477654168635,
370
+ "eval_loss": 0.4395095407962799,
371
+ "eval_mean_token_accuracy": 0.8912856681847278,
372
+ "eval_num_tokens": 504739.0,
373
+ "eval_runtime": 10.6624,
374
+ "eval_samples_per_second": 7.597,
375
+ "eval_steps_per_second": 7.597,
376
+ "step": 300
377
+ },
378
+ {
379
+ "entropy": 0.4189235236495733,
380
+ "epoch": 2.700218818380744,
381
+ "grad_norm": 0.4090176224708557,
382
+ "learning_rate": 5e-06,
383
+ "loss": 0.4158,
384
+ "mean_token_accuracy": 0.9012351542711258,
385
+ "num_tokens": 519973.0,
386
+ "step": 310
387
+ },
388
+ {
389
+ "entropy": 0.4326858527958393,
390
+ "epoch": 2.787746170678337,
391
+ "grad_norm": 0.33089756965637207,
392
+ "learning_rate": 4.730234773653188e-06,
393
+ "loss": 0.4239,
394
+ "mean_token_accuracy": 0.8949055522680283,
395
+ "num_tokens": 536449.0,
396
+ "step": 320
397
+ },
398
+ {
399
+ "entropy": 0.40415228083729743,
400
+ "epoch": 2.87527352297593,
401
+ "grad_norm": 0.2873836159706116,
402
+ "learning_rate": 4.4612553875903e-06,
403
+ "loss": 0.387,
404
+ "mean_token_accuracy": 0.9068056881427765,
405
+ "num_tokens": 552548.0,
406
+ "step": 330
407
+ },
408
+ {
409
+ "entropy": 0.4437519647181034,
410
+ "epoch": 2.962800875273523,
411
+ "grad_norm": 0.3972427546977997,
412
+ "learning_rate": 4.1938453929012014e-06,
413
+ "loss": 0.4373,
414
+ "mean_token_accuracy": 0.8953345134854317,
415
+ "num_tokens": 569790.0,
416
+ "step": 340
417
+ },
418
+ {
419
+ "entropy": 0.39538536683933156,
420
+ "epoch": 3.0437636761487963,
421
+ "grad_norm": 0.27691659331321716,
422
+ "learning_rate": 3.928783768956202e-06,
423
+ "loss": 0.4034,
424
+ "mean_token_accuracy": 0.9004409393748721,
425
+ "num_tokens": 585062.0,
426
+ "step": 350
427
+ },
428
+ {
429
+ "epoch": 3.0437636761487963,
430
+ "eval_entropy": 0.4437022599173181,
431
+ "eval_loss": 0.4359912574291229,
432
+ "eval_mean_token_accuracy": 0.891755164405446,
433
+ "eval_num_tokens": 585062.0,
434
+ "eval_runtime": 10.3036,
435
+ "eval_samples_per_second": 7.861,
436
+ "eval_steps_per_second": 7.861,
437
+ "step": 350
438
+ },
439
+ {
440
+ "entropy": 0.43682731576263906,
441
+ "epoch": 3.1312910284463893,
442
+ "grad_norm": 0.23371583223342896,
443
+ "learning_rate": 3.6668426541981914e-06,
444
+ "loss": 0.426,
445
+ "mean_token_accuracy": 0.8948694601655006,
446
+ "num_tokens": 603636.0,
447
+ "step": 360
448
+ },
449
+ {
450
+ "entropy": 0.43631603941321373,
451
+ "epoch": 3.2188183807439823,
452
+ "grad_norm": 0.30941087007522583,
453
+ "learning_rate": 3.4087850968627823e-06,
454
+ "loss": 0.4239,
455
+ "mean_token_accuracy": 0.8951606392860413,
456
+ "num_tokens": 620904.0,
457
+ "step": 370
458
+ },
459
+ {
460
+ "entropy": 0.43801640942692754,
461
+ "epoch": 3.3063457330415753,
462
+ "grad_norm": 0.2538641095161438,
463
+ "learning_rate": 3.1553628321786746e-06,
464
+ "loss": 0.4534,
465
+ "mean_token_accuracy": 0.89616918861866,
466
+ "num_tokens": 638395.0,
467
+ "step": 380
468
+ },
469
+ {
470
+ "entropy": 0.4042429648339748,
471
+ "epoch": 3.3938730853391683,
472
+ "grad_norm": 0.27114179730415344,
473
+ "learning_rate": 2.907314092523442e-06,
474
+ "loss": 0.3977,
475
+ "mean_token_accuracy": 0.9019500285387039,
476
+ "num_tokens": 655292.0,
477
+ "step": 390
478
+ },
479
+ {
480
+ "entropy": 0.41444903910160064,
481
+ "epoch": 3.4814004376367613,
482
+ "grad_norm": 0.30044931173324585,
483
+ "learning_rate": 2.665361456913797e-06,
484
+ "loss": 0.3897,
485
+ "mean_token_accuracy": 0.9015165165066719,
486
+ "num_tokens": 672299.0,
487
+ "step": 400
488
+ },
489
+ {
490
+ "epoch": 3.4814004376367613,
491
+ "eval_entropy": 0.44009259711077187,
492
+ "eval_loss": 0.43400639295578003,
493
+ "eval_mean_token_accuracy": 0.8921150086838522,
494
+ "eval_num_tokens": 672299.0,
495
+ "eval_runtime": 10.3492,
496
+ "eval_samples_per_second": 7.827,
497
+ "eval_steps_per_second": 7.827,
498
+ "step": 400
499
+ }
500
+ ],
501
+ "logging_steps": 10,
502
+ "max_steps": 600,
503
+ "num_input_tokens_seen": 0,
504
+ "num_train_epochs": 6,
505
+ "save_steps": 100,
506
+ "stateful_callbacks": {
507
+ "TrainerControl": {
508
+ "args": {
509
+ "should_epoch_stop": false,
510
+ "should_evaluate": false,
511
+ "should_log": false,
512
+ "should_save": true,
513
+ "should_training_stop": false
514
+ },
515
+ "attributes": {}
516
+ }
517
+ },
518
+ "total_flos": 3.088887940023091e+16,
519
+ "train_batch_size": 1,
520
+ "trial_name": null,
521
+ "trial_params": null
522
+ }
checkpoint-500/adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/home/support/llm/Ministral-8B-Instruct-2410/",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 64,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "gate_proj",
29
+ "q_proj",
30
+ "k_proj",
31
+ "v_proj",
32
+ "o_proj",
33
+ "down_proj",
34
+ "up_proj"
35
+ ],
36
+ "target_parameters": null,
37
+ "task_type": "CAUSAL_LM",
38
+ "trainable_token_indices": null,
39
+ "use_dora": false,
40
+ "use_qalora": false,
41
+ "use_rslora": false
42
+ }
checkpoint-500/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-500/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7edbeaf20dd7f571b5dd1c54d9ace4f9b6299127cc7ba2afb14a6d51a4a79a4
3
+ size 17078136
checkpoint-500/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-500/trainer_state.json ADDED
@@ -0,0 +1,644 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 500,
3
+ "best_metric": 0.43244463205337524,
4
+ "best_model_checkpoint": "./python-code-testgen-ministral-8B/checkpoint-500",
5
+ "epoch": 4.350109409190372,
6
+ "eval_steps": 50,
7
+ "global_step": 500,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2537571787834167,
14
+ "epoch": 0.087527352297593,
15
+ "grad_norm": 0.5736712217330933,
16
+ "learning_rate": 5e-06,
17
+ "loss": 1.8641,
18
+ "mean_token_accuracy": 0.6149017184972763,
19
+ "num_tokens": 17392.0,
20
+ "step": 10
21
+ },
22
+ {
23
+ "entropy": 1.3263455376029014,
24
+ "epoch": 0.175054704595186,
25
+ "grad_norm": 0.7149288058280945,
26
+ "learning_rate": 9.999927156177032e-06,
27
+ "loss": 1.9444,
28
+ "mean_token_accuracy": 0.601305079460144,
29
+ "num_tokens": 33323.0,
30
+ "step": 20
31
+ },
32
+ {
33
+ "entropy": 1.3946671187877655,
34
+ "epoch": 0.26258205689277897,
35
+ "grad_norm": 0.6501050591468811,
36
+ "learning_rate": 9.991188465340766e-06,
37
+ "loss": 1.7875,
38
+ "mean_token_accuracy": 0.6164133206009865,
39
+ "num_tokens": 49246.0,
40
+ "step": 30
41
+ },
42
+ {
43
+ "entropy": 1.3958179399371147,
44
+ "epoch": 0.350109409190372,
45
+ "grad_norm": 0.5064170360565186,
46
+ "learning_rate": 9.96791018015489e-06,
47
+ "loss": 1.4612,
48
+ "mean_token_accuracy": 0.6549548670649529,
49
+ "num_tokens": 66175.0,
50
+ "step": 40
51
+ },
52
+ {
53
+ "entropy": 1.1895733758807183,
54
+ "epoch": 0.437636761487965,
55
+ "grad_norm": 0.7921415567398071,
56
+ "learning_rate": 9.930160111487717e-06,
57
+ "loss": 1.1449,
58
+ "mean_token_accuracy": 0.7270906791090965,
59
+ "num_tokens": 82566.0,
60
+ "step": 50
61
+ },
62
+ {
63
+ "epoch": 0.437636761487965,
64
+ "eval_entropy": 1.0364094030709914,
65
+ "eval_loss": 1.0214754343032837,
66
+ "eval_mean_token_accuracy": 0.7765023590605936,
67
+ "eval_num_tokens": 82566.0,
68
+ "eval_runtime": 10.3255,
69
+ "eval_samples_per_second": 7.845,
70
+ "eval_steps_per_second": 7.845,
71
+ "step": 50
72
+ },
73
+ {
74
+ "entropy": 0.8925203159451485,
75
+ "epoch": 0.5251641137855579,
76
+ "grad_norm": 0.34061136841773987,
77
+ "learning_rate": 9.87804822727352e-06,
78
+ "loss": 0.9039,
79
+ "mean_token_accuracy": 0.817305064201355,
80
+ "num_tokens": 99237.0,
81
+ "step": 60
82
+ },
83
+ {
84
+ "entropy": 0.6578707128763199,
85
+ "epoch": 0.612691466083151,
86
+ "grad_norm": 0.3180134892463684,
87
+ "learning_rate": 9.811726332170153e-06,
88
+ "loss": 0.7,
89
+ "mean_token_accuracy": 0.8500439867377281,
90
+ "num_tokens": 117524.0,
91
+ "step": 70
92
+ },
93
+ {
94
+ "entropy": 0.5739984951913357,
95
+ "epoch": 0.700218818380744,
96
+ "grad_norm": 0.4566318094730377,
97
+ "learning_rate": 9.731387625344104e-06,
98
+ "loss": 0.6788,
99
+ "mean_token_accuracy": 0.8602568313479424,
100
+ "num_tokens": 135866.0,
101
+ "step": 80
102
+ },
103
+ {
104
+ "entropy": 0.5298320218920708,
105
+ "epoch": 0.787746170678337,
106
+ "grad_norm": 0.31226617097854614,
107
+ "learning_rate": 9.637266137671177e-06,
108
+ "loss": 0.5391,
109
+ "mean_token_accuracy": 0.8844943925738334,
110
+ "num_tokens": 152266.0,
111
+ "step": 90
112
+ },
113
+ {
114
+ "entropy": 0.5451121114194393,
115
+ "epoch": 0.87527352297593,
116
+ "grad_norm": 0.20949594676494598,
117
+ "learning_rate": 9.529636049992235e-06,
118
+ "loss": 0.5172,
119
+ "mean_token_accuracy": 0.8867821574211121,
120
+ "num_tokens": 168374.0,
121
+ "step": 100
122
+ },
123
+ {
124
+ "epoch": 0.87527352297593,
125
+ "eval_entropy": 0.6073500057797373,
126
+ "eval_loss": 0.5493269562721252,
127
+ "eval_mean_token_accuracy": 0.8748943202289534,
128
+ "eval_num_tokens": 168374.0,
129
+ "eval_runtime": 10.3652,
130
+ "eval_samples_per_second": 7.815,
131
+ "eval_steps_per_second": 7.815,
132
+ "step": 100
133
+ },
134
+ {
135
+ "entropy": 0.5541586525738239,
136
+ "epoch": 0.962800875273523,
137
+ "grad_norm": 0.24901992082595825,
138
+ "learning_rate": 9.408810894410008e-06,
139
+ "loss": 0.5106,
140
+ "mean_token_accuracy": 0.8848323300480843,
141
+ "num_tokens": 185723.0,
142
+ "step": 110
143
+ },
144
+ {
145
+ "entropy": 0.5490001190353084,
146
+ "epoch": 1.0437636761487965,
147
+ "grad_norm": 0.2396228015422821,
148
+ "learning_rate": 9.275142640953599e-06,
149
+ "loss": 0.5232,
150
+ "mean_token_accuracy": 0.8872134911047446,
151
+ "num_tokens": 201213.0,
152
+ "step": 120
153
+ },
154
+ {
155
+ "entropy": 0.5084450632333756,
156
+ "epoch": 1.1312910284463895,
157
+ "grad_norm": 0.3080507218837738,
158
+ "learning_rate": 9.129020672271283e-06,
159
+ "loss": 0.464,
160
+ "mean_token_accuracy": 0.8925739243626595,
161
+ "num_tokens": 216879.0,
162
+ "step": 130
163
+ },
164
+ {
165
+ "entropy": 0.47396973669528963,
166
+ "epoch": 1.2188183807439825,
167
+ "grad_norm": 0.2524544894695282,
168
+ "learning_rate": 8.970870649338388e-06,
169
+ "loss": 0.442,
170
+ "mean_token_accuracy": 0.8983234956860542,
171
+ "num_tokens": 233348.0,
172
+ "step": 140
173
+ },
174
+ {
175
+ "entropy": 0.4726570636034012,
176
+ "epoch": 1.3063457330415755,
177
+ "grad_norm": 0.3230877220630646,
178
+ "learning_rate": 8.801153271484503e-06,
179
+ "loss": 0.4538,
180
+ "mean_token_accuracy": 0.8902932360768319,
181
+ "num_tokens": 250578.0,
182
+ "step": 150
183
+ },
184
+ {
185
+ "epoch": 1.3063457330415755,
186
+ "eval_entropy": 0.5106582998493572,
187
+ "eval_loss": 0.4790702760219574,
188
+ "eval_mean_token_accuracy": 0.8854184364095147,
189
+ "eval_num_tokens": 250578.0,
190
+ "eval_runtime": 10.3528,
191
+ "eval_samples_per_second": 7.824,
192
+ "eval_steps_per_second": 7.824,
193
+ "step": 150
194
+ },
195
+ {
196
+ "entropy": 0.48775711581110953,
197
+ "epoch": 1.3938730853391685,
198
+ "grad_norm": 0.27437666058540344,
199
+ "learning_rate": 8.620362934352109e-06,
200
+ "loss": 0.4712,
201
+ "mean_token_accuracy": 0.8893376395106316,
202
+ "num_tokens": 266848.0,
203
+ "step": 160
204
+ },
205
+ {
206
+ "entropy": 0.4436598148196936,
207
+ "epoch": 1.4814004376367615,
208
+ "grad_norm": 0.32482877373695374,
209
+ "learning_rate": 8.429026289696091e-06,
210
+ "loss": 0.4233,
211
+ "mean_token_accuracy": 0.8987479954957962,
212
+ "num_tokens": 282930.0,
213
+ "step": 170
214
+ },
215
+ {
216
+ "entropy": 0.44891218170523645,
217
+ "epoch": 1.5689277899343543,
218
+ "grad_norm": 0.3758986294269562,
219
+ "learning_rate": 8.227700711219492e-06,
220
+ "loss": 0.4315,
221
+ "mean_token_accuracy": 0.8931379184126854,
222
+ "num_tokens": 300722.0,
223
+ "step": 180
224
+ },
225
+ {
226
+ "entropy": 0.5109403282403946,
227
+ "epoch": 1.6564551422319473,
228
+ "grad_norm": 0.3557584583759308,
229
+ "learning_rate": 8.016972670914624e-06,
230
+ "loss": 0.5022,
231
+ "mean_token_accuracy": 0.8783736646175384,
232
+ "num_tokens": 317955.0,
233
+ "step": 190
234
+ },
235
+ {
236
+ "entropy": 0.46201174408197404,
237
+ "epoch": 1.7439824945295404,
238
+ "grad_norm": 0.3397444784641266,
239
+ "learning_rate": 7.797456030639314e-06,
240
+ "loss": 0.4591,
241
+ "mean_token_accuracy": 0.8974948301911354,
242
+ "num_tokens": 334732.0,
243
+ "step": 200
244
+ },
245
+ {
246
+ "epoch": 1.7439824945295404,
247
+ "eval_entropy": 0.47593313657943115,
248
+ "eval_loss": 0.45035651326179504,
249
+ "eval_mean_token_accuracy": 0.8885414180932222,
250
+ "eval_num_tokens": 334732.0,
251
+ "eval_runtime": 10.3648,
252
+ "eval_samples_per_second": 7.815,
253
+ "eval_steps_per_second": 7.815,
254
+ "step": 200
255
+ },
256
+ {
257
+ "entropy": 0.4485505182296038,
258
+ "epoch": 1.8315098468271334,
259
+ "grad_norm": 0.27542436122894287,
260
+ "learning_rate": 7.569790253905059e-06,
261
+ "loss": 0.4558,
262
+ "mean_token_accuracy": 0.8910033285617829,
263
+ "num_tokens": 353384.0,
264
+ "step": 210
265
+ },
266
+ {
267
+ "entropy": 0.450789924710989,
268
+ "epoch": 1.9190371991247264,
269
+ "grad_norm": 0.2546970546245575,
270
+ "learning_rate": 7.334638543086203e-06,
271
+ "loss": 0.4503,
272
+ "mean_token_accuracy": 0.8930477350950241,
273
+ "num_tokens": 370072.0,
274
+ "step": 220
275
+ },
276
+ {
277
+ "entropy": 0.45568384431503917,
278
+ "epoch": 2.0,
279
+ "grad_norm": 0.5460340976715088,
280
+ "learning_rate": 7.092685907476558e-06,
281
+ "loss": 0.4571,
282
+ "mean_token_accuracy": 0.898327117030685,
283
+ "num_tokens": 384804.0,
284
+ "step": 230
285
+ },
286
+ {
287
+ "entropy": 0.44126378260552884,
288
+ "epoch": 2.087527352297593,
289
+ "grad_norm": 0.3868609666824341,
290
+ "learning_rate": 6.844637167821327e-06,
291
+ "loss": 0.423,
292
+ "mean_token_accuracy": 0.8965987235307693,
293
+ "num_tokens": 401856.0,
294
+ "step": 240
295
+ },
296
+ {
297
+ "entropy": 0.42570732310414316,
298
+ "epoch": 2.175054704595186,
299
+ "grad_norm": 0.31367599964141846,
300
+ "learning_rate": 6.591214903137221e-06,
301
+ "loss": 0.4127,
302
+ "mean_token_accuracy": 0.8982817739248276,
303
+ "num_tokens": 418683.0,
304
+ "step": 250
305
+ },
306
+ {
307
+ "epoch": 2.175054704595186,
308
+ "eval_entropy": 0.45187407492855447,
309
+ "eval_loss": 0.44380050897598267,
310
+ "eval_mean_token_accuracy": 0.8904590201966557,
311
+ "eval_num_tokens": 418683.0,
312
+ "eval_runtime": 10.3129,
313
+ "eval_samples_per_second": 7.854,
314
+ "eval_steps_per_second": 7.854,
315
+ "step": 250
316
+ },
317
+ {
318
+ "entropy": 0.4225453745573759,
319
+ "epoch": 2.262582056892779,
320
+ "grad_norm": 0.3352673351764679,
321
+ "learning_rate": 6.333157345801808e-06,
322
+ "loss": 0.4236,
323
+ "mean_token_accuracy": 0.8970558911561965,
324
+ "num_tokens": 435678.0,
325
+ "step": 260
326
+ },
327
+ {
328
+ "entropy": 0.39916136525571344,
329
+ "epoch": 2.350109409190372,
330
+ "grad_norm": 0.3182181119918823,
331
+ "learning_rate": 6.0712162310438e-06,
332
+ "loss": 0.3953,
333
+ "mean_token_accuracy": 0.90475542396307,
334
+ "num_tokens": 453095.0,
335
+ "step": 270
336
+ },
337
+ {
338
+ "entropy": 0.46247223690152167,
339
+ "epoch": 2.437636761487965,
340
+ "grad_norm": 0.27779728174209595,
341
+ "learning_rate": 5.806154607098799e-06,
342
+ "loss": 0.4554,
343
+ "mean_token_accuracy": 0.8864100977778435,
344
+ "num_tokens": 470622.0,
345
+ "step": 280
346
+ },
347
+ {
348
+ "entropy": 0.4407076742500067,
349
+ "epoch": 2.525164113785558,
350
+ "grad_norm": 0.31952330470085144,
351
+ "learning_rate": 5.538744612409701e-06,
352
+ "loss": 0.4446,
353
+ "mean_token_accuracy": 0.8954811543226242,
354
+ "num_tokens": 489165.0,
355
+ "step": 290
356
+ },
357
+ {
358
+ "entropy": 0.4149217315018177,
359
+ "epoch": 2.612691466083151,
360
+ "grad_norm": 0.29808804392814636,
361
+ "learning_rate": 5.269765226346812e-06,
362
+ "loss": 0.4112,
363
+ "mean_token_accuracy": 0.9011785760521889,
364
+ "num_tokens": 504739.0,
365
+ "step": 300
366
+ },
367
+ {
368
+ "epoch": 2.612691466083151,
369
+ "eval_entropy": 0.4445477654168635,
370
+ "eval_loss": 0.4395095407962799,
371
+ "eval_mean_token_accuracy": 0.8912856681847278,
372
+ "eval_num_tokens": 504739.0,
373
+ "eval_runtime": 10.6624,
374
+ "eval_samples_per_second": 7.597,
375
+ "eval_steps_per_second": 7.597,
376
+ "step": 300
377
+ },
378
+ {
379
+ "entropy": 0.4189235236495733,
380
+ "epoch": 2.700218818380744,
381
+ "grad_norm": 0.4090176224708557,
382
+ "learning_rate": 5e-06,
383
+ "loss": 0.4158,
384
+ "mean_token_accuracy": 0.9012351542711258,
385
+ "num_tokens": 519973.0,
386
+ "step": 310
387
+ },
388
+ {
389
+ "entropy": 0.4326858527958393,
390
+ "epoch": 2.787746170678337,
391
+ "grad_norm": 0.33089756965637207,
392
+ "learning_rate": 4.730234773653188e-06,
393
+ "loss": 0.4239,
394
+ "mean_token_accuracy": 0.8949055522680283,
395
+ "num_tokens": 536449.0,
396
+ "step": 320
397
+ },
398
+ {
399
+ "entropy": 0.40415228083729743,
400
+ "epoch": 2.87527352297593,
401
+ "grad_norm": 0.2873836159706116,
402
+ "learning_rate": 4.4612553875903e-06,
403
+ "loss": 0.387,
404
+ "mean_token_accuracy": 0.9068056881427765,
405
+ "num_tokens": 552548.0,
406
+ "step": 330
407
+ },
408
+ {
409
+ "entropy": 0.4437519647181034,
410
+ "epoch": 2.962800875273523,
411
+ "grad_norm": 0.3972427546977997,
412
+ "learning_rate": 4.1938453929012014e-06,
413
+ "loss": 0.4373,
414
+ "mean_token_accuracy": 0.8953345134854317,
415
+ "num_tokens": 569790.0,
416
+ "step": 340
417
+ },
418
+ {
419
+ "entropy": 0.39538536683933156,
420
+ "epoch": 3.0437636761487963,
421
+ "grad_norm": 0.27691659331321716,
422
+ "learning_rate": 3.928783768956202e-06,
423
+ "loss": 0.4034,
424
+ "mean_token_accuracy": 0.9004409393748721,
425
+ "num_tokens": 585062.0,
426
+ "step": 350
427
+ },
428
+ {
429
+ "epoch": 3.0437636761487963,
430
+ "eval_entropy": 0.4437022599173181,
431
+ "eval_loss": 0.4359912574291229,
432
+ "eval_mean_token_accuracy": 0.891755164405446,
433
+ "eval_num_tokens": 585062.0,
434
+ "eval_runtime": 10.3036,
435
+ "eval_samples_per_second": 7.861,
436
+ "eval_steps_per_second": 7.861,
437
+ "step": 350
438
+ },
439
+ {
440
+ "entropy": 0.43682731576263906,
441
+ "epoch": 3.1312910284463893,
442
+ "grad_norm": 0.23371583223342896,
443
+ "learning_rate": 3.6668426541981914e-06,
444
+ "loss": 0.426,
445
+ "mean_token_accuracy": 0.8948694601655006,
446
+ "num_tokens": 603636.0,
447
+ "step": 360
448
+ },
449
+ {
450
+ "entropy": 0.43631603941321373,
451
+ "epoch": 3.2188183807439823,
452
+ "grad_norm": 0.30941087007522583,
453
+ "learning_rate": 3.4087850968627823e-06,
454
+ "loss": 0.4239,
455
+ "mean_token_accuracy": 0.8951606392860413,
456
+ "num_tokens": 620904.0,
457
+ "step": 370
458
+ },
459
+ {
460
+ "entropy": 0.43801640942692754,
461
+ "epoch": 3.3063457330415753,
462
+ "grad_norm": 0.2538641095161438,
463
+ "learning_rate": 3.1553628321786746e-06,
464
+ "loss": 0.4534,
465
+ "mean_token_accuracy": 0.89616918861866,
466
+ "num_tokens": 638395.0,
467
+ "step": 380
468
+ },
469
+ {
470
+ "entropy": 0.4042429648339748,
471
+ "epoch": 3.3938730853391683,
472
+ "grad_norm": 0.27114179730415344,
473
+ "learning_rate": 2.907314092523442e-06,
474
+ "loss": 0.3977,
475
+ "mean_token_accuracy": 0.9019500285387039,
476
+ "num_tokens": 655292.0,
477
+ "step": 390
478
+ },
479
+ {
480
+ "entropy": 0.41444903910160064,
481
+ "epoch": 3.4814004376367613,
482
+ "grad_norm": 0.30044931173324585,
483
+ "learning_rate": 2.665361456913797e-06,
484
+ "loss": 0.3897,
485
+ "mean_token_accuracy": 0.9015165165066719,
486
+ "num_tokens": 672299.0,
487
+ "step": 400
488
+ },
489
+ {
490
+ "epoch": 3.4814004376367613,
491
+ "eval_entropy": 0.44009259711077187,
492
+ "eval_loss": 0.43400639295578003,
493
+ "eval_mean_token_accuracy": 0.8921150086838522,
494
+ "eval_num_tokens": 672299.0,
495
+ "eval_runtime": 10.3492,
496
+ "eval_samples_per_second": 7.827,
497
+ "eval_steps_per_second": 7.827,
498
+ "step": 400
499
+ },
500
+ {
501
+ "entropy": 0.4128158770501614,
502
+ "epoch": 3.5689277899343543,
503
+ "grad_norm": 0.3554251194000244,
504
+ "learning_rate": 2.430209746094943e-06,
505
+ "loss": 0.4109,
506
+ "mean_token_accuracy": 0.9037208989262581,
507
+ "num_tokens": 689112.0,
508
+ "step": 410
509
+ },
510
+ {
511
+ "entropy": 0.4040897026658058,
512
+ "epoch": 3.6564551422319473,
513
+ "grad_norm": 0.27590811252593994,
514
+ "learning_rate": 2.202543969360688e-06,
515
+ "loss": 0.4036,
516
+ "mean_token_accuracy": 0.9008671209216118,
517
+ "num_tokens": 706000.0,
518
+ "step": 420
519
+ },
520
+ {
521
+ "entropy": 0.3941893048584461,
522
+ "epoch": 3.7439824945295404,
523
+ "grad_norm": 0.30366408824920654,
524
+ "learning_rate": 1.983027329085377e-06,
525
+ "loss": 0.3765,
526
+ "mean_token_accuracy": 0.9078722298145294,
527
+ "num_tokens": 722208.0,
528
+ "step": 430
529
+ },
530
+ {
531
+ "entropy": 0.4085026063024998,
532
+ "epoch": 3.8315098468271334,
533
+ "grad_norm": 0.35508859157562256,
534
+ "learning_rate": 1.772299288780508e-06,
535
+ "loss": 0.3974,
536
+ "mean_token_accuracy": 0.8996173903346062,
537
+ "num_tokens": 738709.0,
538
+ "step": 440
539
+ },
540
+ {
541
+ "entropy": 0.44988039061427115,
542
+ "epoch": 3.9190371991247264,
543
+ "grad_norm": 0.3549656867980957,
544
+ "learning_rate": 1.5709737103039101e-06,
545
+ "loss": 0.4376,
546
+ "mean_token_accuracy": 0.8958355247974396,
547
+ "num_tokens": 754786.0,
548
+ "step": 450
549
+ },
550
+ {
551
+ "epoch": 3.9190371991247264,
552
+ "eval_entropy": 0.43313855797420314,
553
+ "eval_loss": 0.433157354593277,
554
+ "eval_mean_token_accuracy": 0.8922798493762075,
555
+ "eval_num_tokens": 754786.0,
556
+ "eval_runtime": 10.316,
557
+ "eval_samples_per_second": 7.852,
558
+ "eval_steps_per_second": 7.852,
559
+ "step": 450
560
+ },
561
+ {
562
+ "entropy": 0.40361173571767034,
563
+ "epoch": 4.0,
564
+ "grad_norm": 0.5634493231773376,
565
+ "learning_rate": 1.3796370656478936e-06,
566
+ "loss": 0.3918,
567
+ "mean_token_accuracy": 0.9029259810576568,
568
+ "num_tokens": 769608.0,
569
+ "step": 460
570
+ },
571
+ {
572
+ "entropy": 0.43113911151885986,
573
+ "epoch": 4.087527352297593,
574
+ "grad_norm": 0.27384519577026367,
575
+ "learning_rate": 1.1988467285154987e-06,
576
+ "loss": 0.4191,
577
+ "mean_token_accuracy": 0.898203282058239,
578
+ "num_tokens": 786018.0,
579
+ "step": 470
580
+ },
581
+ {
582
+ "entropy": 0.42219107821583746,
583
+ "epoch": 4.175054704595186,
584
+ "grad_norm": 0.36136895418167114,
585
+ "learning_rate": 1.0291293506616134e-06,
586
+ "loss": 0.4265,
587
+ "mean_token_accuracy": 0.8964347586035728,
588
+ "num_tokens": 802910.0,
589
+ "step": 480
590
+ },
591
+ {
592
+ "entropy": 0.4038095213472843,
593
+ "epoch": 4.262582056892779,
594
+ "grad_norm": 0.3551323413848877,
595
+ "learning_rate": 8.709793277287182e-07,
596
+ "loss": 0.3885,
597
+ "mean_token_accuracy": 0.9044827505946159,
598
+ "num_tokens": 819640.0,
599
+ "step": 490
600
+ },
601
+ {
602
+ "entropy": 0.36865628361701963,
603
+ "epoch": 4.350109409190372,
604
+ "grad_norm": 0.2690926790237427,
605
+ "learning_rate": 7.248573590464015e-07,
606
+ "loss": 0.3695,
607
+ "mean_token_accuracy": 0.9067655742168427,
608
+ "num_tokens": 836913.0,
609
+ "step": 500
610
+ },
611
+ {
612
+ "epoch": 4.350109409190372,
613
+ "eval_entropy": 0.43300218015541264,
614
+ "eval_loss": 0.43244463205337524,
615
+ "eval_mean_token_accuracy": 0.8924668644681389,
616
+ "eval_num_tokens": 836913.0,
617
+ "eval_runtime": 10.385,
618
+ "eval_samples_per_second": 7.8,
619
+ "eval_steps_per_second": 7.8,
620
+ "step": 500
621
+ }
622
+ ],
623
+ "logging_steps": 10,
624
+ "max_steps": 600,
625
+ "num_input_tokens_seen": 0,
626
+ "num_train_epochs": 6,
627
+ "save_steps": 100,
628
+ "stateful_callbacks": {
629
+ "TrainerControl": {
630
+ "args": {
631
+ "should_epoch_stop": false,
632
+ "should_evaluate": false,
633
+ "should_log": false,
634
+ "should_save": true,
635
+ "should_training_stop": false
636
+ },
637
+ "attributes": {}
638
+ }
639
+ },
640
+ "total_flos": 3.845209456727654e+16,
641
+ "train_batch_size": 1,
642
+ "trial_name": null,
643
+ "trial_params": null
644
+ }
checkpoint-600/adapter_config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/home/support/llm/Ministral-8B-Instruct-2410/",
5
+ "bias": "none",
6
+ "corda_config": null,
7
+ "eva_config": null,
8
+ "exclude_modules": null,
9
+ "fan_in_fan_out": false,
10
+ "inference_mode": true,
11
+ "init_lora_weights": true,
12
+ "layer_replication": null,
13
+ "layers_pattern": null,
14
+ "layers_to_transform": null,
15
+ "loftq_config": {},
16
+ "lora_alpha": 16,
17
+ "lora_bias": false,
18
+ "lora_dropout": 0.05,
19
+ "megatron_config": null,
20
+ "megatron_core": "megatron.core",
21
+ "modules_to_save": null,
22
+ "peft_type": "LORA",
23
+ "qalora_group_size": 16,
24
+ "r": 64,
25
+ "rank_pattern": {},
26
+ "revision": null,
27
+ "target_modules": [
28
+ "gate_proj",
29
+ "q_proj",
30
+ "k_proj",
31
+ "v_proj",
32
+ "o_proj",
33
+ "down_proj",
34
+ "up_proj"
35
+ ],
36
+ "target_parameters": null,
37
+ "task_type": "CAUSAL_LM",
38
+ "trainable_token_indices": null,
39
+ "use_dora": false,
40
+ "use_qalora": false,
41
+ "use_rslora": false
42
+ }
checkpoint-600/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-600/tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7edbeaf20dd7f571b5dd1c54d9ace4f9b6299127cc7ba2afb14a6d51a4a79a4
3
+ size 17078136
checkpoint-600/tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff
 
checkpoint-600/trainer_state.json ADDED
@@ -0,0 +1,766 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": 600,
3
+ "best_metric": 0.4320242702960968,
4
+ "best_model_checkpoint": "./python-code-testgen-ministral-8B/checkpoint-600",
5
+ "epoch": 5.218818380743983,
6
+ "eval_steps": 50,
7
+ "global_step": 600,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "entropy": 1.2537571787834167,
14
+ "epoch": 0.087527352297593,
15
+ "grad_norm": 0.5736712217330933,
16
+ "learning_rate": 5e-06,
17
+ "loss": 1.8641,
18
+ "mean_token_accuracy": 0.6149017184972763,
19
+ "num_tokens": 17392.0,
20
+ "step": 10
21
+ },
22
+ {
23
+ "entropy": 1.3263455376029014,
24
+ "epoch": 0.175054704595186,
25
+ "grad_norm": 0.7149288058280945,
26
+ "learning_rate": 9.999927156177032e-06,
27
+ "loss": 1.9444,
28
+ "mean_token_accuracy": 0.601305079460144,
29
+ "num_tokens": 33323.0,
30
+ "step": 20
31
+ },
32
+ {
33
+ "entropy": 1.3946671187877655,
34
+ "epoch": 0.26258205689277897,
35
+ "grad_norm": 0.6501050591468811,
36
+ "learning_rate": 9.991188465340766e-06,
37
+ "loss": 1.7875,
38
+ "mean_token_accuracy": 0.6164133206009865,
39
+ "num_tokens": 49246.0,
40
+ "step": 30
41
+ },
42
+ {
43
+ "entropy": 1.3958179399371147,
44
+ "epoch": 0.350109409190372,
45
+ "grad_norm": 0.5064170360565186,
46
+ "learning_rate": 9.96791018015489e-06,
47
+ "loss": 1.4612,
48
+ "mean_token_accuracy": 0.6549548670649529,
49
+ "num_tokens": 66175.0,
50
+ "step": 40
51
+ },
52
+ {
53
+ "entropy": 1.1895733758807183,
54
+ "epoch": 0.437636761487965,
55
+ "grad_norm": 0.7921415567398071,
56
+ "learning_rate": 9.930160111487717e-06,
57
+ "loss": 1.1449,
58
+ "mean_token_accuracy": 0.7270906791090965,
59
+ "num_tokens": 82566.0,
60
+ "step": 50
61
+ },
62
+ {
63
+ "epoch": 0.437636761487965,
64
+ "eval_entropy": 1.0364094030709914,
65
+ "eval_loss": 1.0214754343032837,
66
+ "eval_mean_token_accuracy": 0.7765023590605936,
67
+ "eval_num_tokens": 82566.0,
68
+ "eval_runtime": 10.3255,
69
+ "eval_samples_per_second": 7.845,
70
+ "eval_steps_per_second": 7.845,
71
+ "step": 50
72
+ },
73
+ {
74
+ "entropy": 0.8925203159451485,
75
+ "epoch": 0.5251641137855579,
76
+ "grad_norm": 0.34061136841773987,
77
+ "learning_rate": 9.87804822727352e-06,
78
+ "loss": 0.9039,
79
+ "mean_token_accuracy": 0.817305064201355,
80
+ "num_tokens": 99237.0,
81
+ "step": 60
82
+ },
83
+ {
84
+ "entropy": 0.6578707128763199,
85
+ "epoch": 0.612691466083151,
86
+ "grad_norm": 0.3180134892463684,
87
+ "learning_rate": 9.811726332170153e-06,
88
+ "loss": 0.7,
89
+ "mean_token_accuracy": 0.8500439867377281,
90
+ "num_tokens": 117524.0,
91
+ "step": 70
92
+ },
93
+ {
94
+ "entropy": 0.5739984951913357,
95
+ "epoch": 0.700218818380744,
96
+ "grad_norm": 0.4566318094730377,
97
+ "learning_rate": 9.731387625344104e-06,
98
+ "loss": 0.6788,
99
+ "mean_token_accuracy": 0.8602568313479424,
100
+ "num_tokens": 135866.0,
101
+ "step": 80
102
+ },
103
+ {
104
+ "entropy": 0.5298320218920708,
105
+ "epoch": 0.787746170678337,
106
+ "grad_norm": 0.31226617097854614,
107
+ "learning_rate": 9.637266137671177e-06,
108
+ "loss": 0.5391,
109
+ "mean_token_accuracy": 0.8844943925738334,
110
+ "num_tokens": 152266.0,
111
+ "step": 90
112
+ },
113
+ {
114
+ "entropy": 0.5451121114194393,
115
+ "epoch": 0.87527352297593,
116
+ "grad_norm": 0.20949594676494598,
117
+ "learning_rate": 9.529636049992235e-06,
118
+ "loss": 0.5172,
119
+ "mean_token_accuracy": 0.8867821574211121,
120
+ "num_tokens": 168374.0,
121
+ "step": 100
122
+ },
123
+ {
124
+ "epoch": 0.87527352297593,
125
+ "eval_entropy": 0.6073500057797373,
126
+ "eval_loss": 0.5493269562721252,
127
+ "eval_mean_token_accuracy": 0.8748943202289534,
128
+ "eval_num_tokens": 168374.0,
129
+ "eval_runtime": 10.3652,
130
+ "eval_samples_per_second": 7.815,
131
+ "eval_steps_per_second": 7.815,
132
+ "step": 100
133
+ },
134
+ {
135
+ "entropy": 0.5541586525738239,
136
+ "epoch": 0.962800875273523,
137
+ "grad_norm": 0.24901992082595825,
138
+ "learning_rate": 9.408810894410008e-06,
139
+ "loss": 0.5106,
140
+ "mean_token_accuracy": 0.8848323300480843,
141
+ "num_tokens": 185723.0,
142
+ "step": 110
143
+ },
144
+ {
145
+ "entropy": 0.5490001190353084,
146
+ "epoch": 1.0437636761487965,
147
+ "grad_norm": 0.2396228015422821,
148
+ "learning_rate": 9.275142640953599e-06,
149
+ "loss": 0.5232,
150
+ "mean_token_accuracy": 0.8872134911047446,
151
+ "num_tokens": 201213.0,
152
+ "step": 120
153
+ },
154
+ {
155
+ "entropy": 0.5084450632333756,
156
+ "epoch": 1.1312910284463895,
157
+ "grad_norm": 0.3080507218837738,
158
+ "learning_rate": 9.129020672271283e-06,
159
+ "loss": 0.464,
160
+ "mean_token_accuracy": 0.8925739243626595,
161
+ "num_tokens": 216879.0,
162
+ "step": 130
163
+ },
164
+ {
165
+ "entropy": 0.47396973669528963,
166
+ "epoch": 1.2188183807439825,
167
+ "grad_norm": 0.2524544894695282,
168
+ "learning_rate": 8.970870649338388e-06,
169
+ "loss": 0.442,
170
+ "mean_token_accuracy": 0.8983234956860542,
171
+ "num_tokens": 233348.0,
172
+ "step": 140
173
+ },
174
+ {
175
+ "entropy": 0.4726570636034012,
176
+ "epoch": 1.3063457330415755,
177
+ "grad_norm": 0.3230877220630646,
178
+ "learning_rate": 8.801153271484503e-06,
179
+ "loss": 0.4538,
180
+ "mean_token_accuracy": 0.8902932360768319,
181
+ "num_tokens": 250578.0,
182
+ "step": 150
183
+ },
184
+ {
185
+ "epoch": 1.3063457330415755,
186
+ "eval_entropy": 0.5106582998493572,
187
+ "eval_loss": 0.4790702760219574,
188
+ "eval_mean_token_accuracy": 0.8854184364095147,
189
+ "eval_num_tokens": 250578.0,
190
+ "eval_runtime": 10.3528,
191
+ "eval_samples_per_second": 7.824,
192
+ "eval_steps_per_second": 7.824,
193
+ "step": 150
194
+ },
195
+ {
196
+ "entropy": 0.48775711581110953,
197
+ "epoch": 1.3938730853391685,
198
+ "grad_norm": 0.27437666058540344,
199
+ "learning_rate": 8.620362934352109e-06,
200
+ "loss": 0.4712,
201
+ "mean_token_accuracy": 0.8893376395106316,
202
+ "num_tokens": 266848.0,
203
+ "step": 160
204
+ },
205
+ {
206
+ "entropy": 0.4436598148196936,
207
+ "epoch": 1.4814004376367615,
208
+ "grad_norm": 0.32482877373695374,
209
+ "learning_rate": 8.429026289696091e-06,
210
+ "loss": 0.4233,
211
+ "mean_token_accuracy": 0.8987479954957962,
212
+ "num_tokens": 282930.0,
213
+ "step": 170
214
+ },
215
+ {
216
+ "entropy": 0.44891218170523645,
217
+ "epoch": 1.5689277899343543,
218
+ "grad_norm": 0.3758986294269562,
219
+ "learning_rate": 8.227700711219492e-06,
220
+ "loss": 0.4315,
221
+ "mean_token_accuracy": 0.8931379184126854,
222
+ "num_tokens": 300722.0,
223
+ "step": 180
224
+ },
225
+ {
226
+ "entropy": 0.5109403282403946,
227
+ "epoch": 1.6564551422319473,
228
+ "grad_norm": 0.3557584583759308,
229
+ "learning_rate": 8.016972670914624e-06,
230
+ "loss": 0.5022,
231
+ "mean_token_accuracy": 0.8783736646175384,
232
+ "num_tokens": 317955.0,
233
+ "step": 190
234
+ },
235
+ {
236
+ "entropy": 0.46201174408197404,
237
+ "epoch": 1.7439824945295404,
238
+ "grad_norm": 0.3397444784641266,
239
+ "learning_rate": 7.797456030639314e-06,
240
+ "loss": 0.4591,
241
+ "mean_token_accuracy": 0.8974948301911354,
242
+ "num_tokens": 334732.0,
243
+ "step": 200
244
+ },
245
+ {
246
+ "epoch": 1.7439824945295404,
247
+ "eval_entropy": 0.47593313657943115,
248
+ "eval_loss": 0.45035651326179504,
249
+ "eval_mean_token_accuracy": 0.8885414180932222,
250
+ "eval_num_tokens": 334732.0,
251
+ "eval_runtime": 10.3648,
252
+ "eval_samples_per_second": 7.815,
253
+ "eval_steps_per_second": 7.815,
254
+ "step": 200
255
+ },
256
+ {
257
+ "entropy": 0.4485505182296038,
258
+ "epoch": 1.8315098468271334,
259
+ "grad_norm": 0.27542436122894287,
260
+ "learning_rate": 7.569790253905059e-06,
261
+ "loss": 0.4558,
262
+ "mean_token_accuracy": 0.8910033285617829,
263
+ "num_tokens": 353384.0,
264
+ "step": 210
265
+ },
266
+ {
267
+ "entropy": 0.450789924710989,
268
+ "epoch": 1.9190371991247264,
269
+ "grad_norm": 0.2546970546245575,
270
+ "learning_rate": 7.334638543086203e-06,
271
+ "loss": 0.4503,
272
+ "mean_token_accuracy": 0.8930477350950241,
273
+ "num_tokens": 370072.0,
274
+ "step": 220
275
+ },
276
+ {
277
+ "entropy": 0.45568384431503917,
278
+ "epoch": 2.0,
279
+ "grad_norm": 0.5460340976715088,
280
+ "learning_rate": 7.092685907476558e-06,
281
+ "loss": 0.4571,
282
+ "mean_token_accuracy": 0.898327117030685,
283
+ "num_tokens": 384804.0,
284
+ "step": 230
285
+ },
286
+ {
287
+ "entropy": 0.44126378260552884,
288
+ "epoch": 2.087527352297593,
289
+ "grad_norm": 0.3868609666824341,
290
+ "learning_rate": 6.844637167821327e-06,
291
+ "loss": 0.423,
292
+ "mean_token_accuracy": 0.8965987235307693,
293
+ "num_tokens": 401856.0,
294
+ "step": 240
295
+ },
296
+ {
297
+ "entropy": 0.42570732310414316,
298
+ "epoch": 2.175054704595186,
299
+ "grad_norm": 0.31367599964141846,
300
+ "learning_rate": 6.591214903137221e-06,
301
+ "loss": 0.4127,
302
+ "mean_token_accuracy": 0.8982817739248276,
303
+ "num_tokens": 418683.0,
304
+ "step": 250
305
+ },
306
+ {
307
+ "epoch": 2.175054704595186,
308
+ "eval_entropy": 0.45187407492855447,
309
+ "eval_loss": 0.44380050897598267,
310
+ "eval_mean_token_accuracy": 0.8904590201966557,
311
+ "eval_num_tokens": 418683.0,
312
+ "eval_runtime": 10.3129,
313
+ "eval_samples_per_second": 7.854,
314
+ "eval_steps_per_second": 7.854,
315
+ "step": 250
316
+ },
317
+ {
318
+ "entropy": 0.4225453745573759,
319
+ "epoch": 2.262582056892779,
320
+ "grad_norm": 0.3352673351764679,
321
+ "learning_rate": 6.333157345801808e-06,
322
+ "loss": 0.4236,
323
+ "mean_token_accuracy": 0.8970558911561965,
324
+ "num_tokens": 435678.0,
325
+ "step": 260
326
+ },
327
+ {
328
+ "entropy": 0.39916136525571344,
329
+ "epoch": 2.350109409190372,
330
+ "grad_norm": 0.3182181119918823,
331
+ "learning_rate": 6.0712162310438e-06,
332
+ "loss": 0.3953,
333
+ "mean_token_accuracy": 0.90475542396307,
334
+ "num_tokens": 453095.0,
335
+ "step": 270
336
+ },
337
+ {
338
+ "entropy": 0.46247223690152167,
339
+ "epoch": 2.437636761487965,
340
+ "grad_norm": 0.27779728174209595,
341
+ "learning_rate": 5.806154607098799e-06,
342
+ "loss": 0.4554,
343
+ "mean_token_accuracy": 0.8864100977778435,
344
+ "num_tokens": 470622.0,
345
+ "step": 280
346
+ },
347
+ {
348
+ "entropy": 0.4407076742500067,
349
+ "epoch": 2.525164113785558,
350
+ "grad_norm": 0.31952330470085144,
351
+ "learning_rate": 5.538744612409701e-06,
352
+ "loss": 0.4446,
353
+ "mean_token_accuracy": 0.8954811543226242,
354
+ "num_tokens": 489165.0,
355
+ "step": 290
356
+ },
357
+ {
358
+ "entropy": 0.4149217315018177,
359
+ "epoch": 2.612691466083151,
360
+ "grad_norm": 0.29808804392814636,
361
+ "learning_rate": 5.269765226346812e-06,
362
+ "loss": 0.4112,
363
+ "mean_token_accuracy": 0.9011785760521889,
364
+ "num_tokens": 504739.0,
365
+ "step": 300
366
+ },
367
+ {
368
+ "epoch": 2.612691466083151,
369
+ "eval_entropy": 0.4445477654168635,
370
+ "eval_loss": 0.4395095407962799,
371
+ "eval_mean_token_accuracy": 0.8912856681847278,
372
+ "eval_num_tokens": 504739.0,
373
+ "eval_runtime": 10.6624,
374
+ "eval_samples_per_second": 7.597,
375
+ "eval_steps_per_second": 7.597,
376
+ "step": 300
377
+ },
378
+ {
379
+ "entropy": 0.4189235236495733,
380
+ "epoch": 2.700218818380744,
381
+ "grad_norm": 0.4090176224708557,
382
+ "learning_rate": 5e-06,
383
+ "loss": 0.4158,
384
+ "mean_token_accuracy": 0.9012351542711258,
385
+ "num_tokens": 519973.0,
386
+ "step": 310
387
+ },
388
+ {
389
+ "entropy": 0.4326858527958393,
390
+ "epoch": 2.787746170678337,
391
+ "grad_norm": 0.33089756965637207,
392
+ "learning_rate": 4.730234773653188e-06,
393
+ "loss": 0.4239,
394
+ "mean_token_accuracy": 0.8949055522680283,
395
+ "num_tokens": 536449.0,
396
+ "step": 320
397
+ },
398
+ {
399
+ "entropy": 0.40415228083729743,
400
+ "epoch": 2.87527352297593,
401
+ "grad_norm": 0.2873836159706116,
402
+ "learning_rate": 4.4612553875903e-06,
403
+ "loss": 0.387,
404
+ "mean_token_accuracy": 0.9068056881427765,
405
+ "num_tokens": 552548.0,
406
+ "step": 330
407
+ },
408
+ {
409
+ "entropy": 0.4437519647181034,
410
+ "epoch": 2.962800875273523,
411
+ "grad_norm": 0.3972427546977997,
412
+ "learning_rate": 4.1938453929012014e-06,
413
+ "loss": 0.4373,
414
+ "mean_token_accuracy": 0.8953345134854317,
415
+ "num_tokens": 569790.0,
416
+ "step": 340
417
+ },
418
+ {
419
+ "entropy": 0.39538536683933156,
420
+ "epoch": 3.0437636761487963,
421
+ "grad_norm": 0.27691659331321716,
422
+ "learning_rate": 3.928783768956202e-06,
423
+ "loss": 0.4034,
424
+ "mean_token_accuracy": 0.9004409393748721,
425
+ "num_tokens": 585062.0,
426
+ "step": 350
427
+ },
428
+ {
429
+ "epoch": 3.0437636761487963,
430
+ "eval_entropy": 0.4437022599173181,
431
+ "eval_loss": 0.4359912574291229,
432
+ "eval_mean_token_accuracy": 0.891755164405446,
433
+ "eval_num_tokens": 585062.0,
434
+ "eval_runtime": 10.3036,
435
+ "eval_samples_per_second": 7.861,
436
+ "eval_steps_per_second": 7.861,
437
+ "step": 350
438
+ },
439
+ {
440
+ "entropy": 0.43682731576263906,
441
+ "epoch": 3.1312910284463893,
442
+ "grad_norm": 0.23371583223342896,
443
+ "learning_rate": 3.6668426541981914e-06,
444
+ "loss": 0.426,
445
+ "mean_token_accuracy": 0.8948694601655006,
446
+ "num_tokens": 603636.0,
447
+ "step": 360
448
+ },
449
+ {
450
+ "entropy": 0.43631603941321373,
451
+ "epoch": 3.2188183807439823,
452
+ "grad_norm": 0.30941087007522583,
453
+ "learning_rate": 3.4087850968627823e-06,
454
+ "loss": 0.4239,
455
+ "mean_token_accuracy": 0.8951606392860413,
456
+ "num_tokens": 620904.0,
457
+ "step": 370
458
+ },
459
+ {
460
+ "entropy": 0.43801640942692754,
461
+ "epoch": 3.3063457330415753,
462
+ "grad_norm": 0.2538641095161438,
463
+ "learning_rate": 3.1553628321786746e-06,
464
+ "loss": 0.4534,
465
+ "mean_token_accuracy": 0.89616918861866,
466
+ "num_tokens": 638395.0,
467
+ "step": 380
468
+ },
469
+ {
470
+ "entropy": 0.4042429648339748,
471
+ "epoch": 3.3938730853391683,
472
+ "grad_norm": 0.27114179730415344,
473
+ "learning_rate": 2.907314092523442e-06,
474
+ "loss": 0.3977,
475
+ "mean_token_accuracy": 0.9019500285387039,
476
+ "num_tokens": 655292.0,
477
+ "step": 390
478
+ },
479
+ {
480
+ "entropy": 0.41444903910160064,
481
+ "epoch": 3.4814004376367613,
482
+ "grad_norm": 0.30044931173324585,
483
+ "learning_rate": 2.665361456913797e-06,
484
+ "loss": 0.3897,
485
+ "mean_token_accuracy": 0.9015165165066719,
486
+ "num_tokens": 672299.0,
487
+ "step": 400
488
+ },
489
+ {
490
+ "epoch": 3.4814004376367613,
491
+ "eval_entropy": 0.44009259711077187,
492
+ "eval_loss": 0.43400639295578003,
493
+ "eval_mean_token_accuracy": 0.8921150086838522,
494
+ "eval_num_tokens": 672299.0,
495
+ "eval_runtime": 10.3492,
496
+ "eval_samples_per_second": 7.827,
497
+ "eval_steps_per_second": 7.827,
498
+ "step": 400
499
+ },
500
+ {
501
+ "entropy": 0.4128158770501614,
502
+ "epoch": 3.5689277899343543,
503
+ "grad_norm": 0.3554251194000244,
504
+ "learning_rate": 2.430209746094943e-06,
505
+ "loss": 0.4109,
506
+ "mean_token_accuracy": 0.9037208989262581,
507
+ "num_tokens": 689112.0,
508
+ "step": 410
509
+ },
510
+ {
511
+ "entropy": 0.4040897026658058,
512
+ "epoch": 3.6564551422319473,
513
+ "grad_norm": 0.27590811252593994,
514
+ "learning_rate": 2.202543969360688e-06,
515
+ "loss": 0.4036,
516
+ "mean_token_accuracy": 0.9008671209216118,
517
+ "num_tokens": 706000.0,
518
+ "step": 420
519
+ },
520
+ {
521
+ "entropy": 0.3941893048584461,
522
+ "epoch": 3.7439824945295404,
523
+ "grad_norm": 0.30366408824920654,
524
+ "learning_rate": 1.983027329085377e-06,
525
+ "loss": 0.3765,
526
+ "mean_token_accuracy": 0.9078722298145294,
527
+ "num_tokens": 722208.0,
528
+ "step": 430
529
+ },
530
+ {
531
+ "entropy": 0.4085026063024998,
532
+ "epoch": 3.8315098468271334,
533
+ "grad_norm": 0.35508859157562256,
534
+ "learning_rate": 1.772299288780508e-06,
535
+ "loss": 0.3974,
536
+ "mean_token_accuracy": 0.8996173903346062,
537
+ "num_tokens": 738709.0,
538
+ "step": 440
539
+ },
540
+ {
541
+ "entropy": 0.44988039061427115,
542
+ "epoch": 3.9190371991247264,
543
+ "grad_norm": 0.3549656867980957,
544
+ "learning_rate": 1.5709737103039101e-06,
545
+ "loss": 0.4376,
546
+ "mean_token_accuracy": 0.8958355247974396,
547
+ "num_tokens": 754786.0,
548
+ "step": 450
549
+ },
550
+ {
551
+ "epoch": 3.9190371991247264,
552
+ "eval_entropy": 0.43313855797420314,
553
+ "eval_loss": 0.433157354593277,
554
+ "eval_mean_token_accuracy": 0.8922798493762075,
555
+ "eval_num_tokens": 754786.0,
556
+ "eval_runtime": 10.316,
557
+ "eval_samples_per_second": 7.852,
558
+ "eval_steps_per_second": 7.852,
559
+ "step": 450
560
+ },
561
+ {
562
+ "entropy": 0.40361173571767034,
563
+ "epoch": 4.0,
564
+ "grad_norm": 0.5634493231773376,
565
+ "learning_rate": 1.3796370656478936e-06,
566
+ "loss": 0.3918,
567
+ "mean_token_accuracy": 0.9029259810576568,
568
+ "num_tokens": 769608.0,
569
+ "step": 460
570
+ },
571
+ {
572
+ "entropy": 0.43113911151885986,
573
+ "epoch": 4.087527352297593,
574
+ "grad_norm": 0.27384519577026367,
575
+ "learning_rate": 1.1988467285154987e-06,
576
+ "loss": 0.4191,
577
+ "mean_token_accuracy": 0.898203282058239,
578
+ "num_tokens": 786018.0,
579
+ "step": 470
580
+ },
581
+ {
582
+ "entropy": 0.42219107821583746,
583
+ "epoch": 4.175054704595186,
584
+ "grad_norm": 0.36136895418167114,
585
+ "learning_rate": 1.0291293506616134e-06,
586
+ "loss": 0.4265,
587
+ "mean_token_accuracy": 0.8964347586035728,
588
+ "num_tokens": 802910.0,
589
+ "step": 480
590
+ },
591
+ {
592
+ "entropy": 0.4038095213472843,
593
+ "epoch": 4.262582056892779,
594
+ "grad_norm": 0.3551323413848877,
595
+ "learning_rate": 8.709793277287182e-07,
596
+ "loss": 0.3885,
597
+ "mean_token_accuracy": 0.9044827505946159,
598
+ "num_tokens": 819640.0,
599
+ "step": 490
600
+ },
601
+ {
602
+ "entropy": 0.36865628361701963,
603
+ "epoch": 4.350109409190372,
604
+ "grad_norm": 0.2690926790237427,
605
+ "learning_rate": 7.248573590464015e-07,
606
+ "loss": 0.3695,
607
+ "mean_token_accuracy": 0.9067655742168427,
608
+ "num_tokens": 836913.0,
609
+ "step": 500
610
+ },
611
+ {
612
+ "epoch": 4.350109409190372,
613
+ "eval_entropy": 0.43300218015541264,
614
+ "eval_loss": 0.43244463205337524,
615
+ "eval_mean_token_accuracy": 0.8924668644681389,
616
+ "eval_num_tokens": 836913.0,
617
+ "eval_runtime": 10.385,
618
+ "eval_samples_per_second": 7.8,
619
+ "eval_steps_per_second": 7.8,
620
+ "step": 500
621
+ },
622
+ {
623
+ "entropy": 0.3909595631062984,
624
+ "epoch": 4.437636761487965,
625
+ "grad_norm": 0.290950745344162,
626
+ "learning_rate": 5.911891055899921e-07,
627
+ "loss": 0.3792,
628
+ "mean_token_accuracy": 0.9053617954254151,
629
+ "num_tokens": 853221.0,
630
+ "step": 510
631
+ },
632
+ {
633
+ "entropy": 0.3872651342302561,
634
+ "epoch": 4.525164113785558,
635
+ "grad_norm": 0.30887123942375183,
636
+ "learning_rate": 4.7036395000776556e-07,
637
+ "loss": 0.3805,
638
+ "mean_token_accuracy": 0.9081572830677033,
639
+ "num_tokens": 869573.0,
640
+ "step": 520
641
+ },
642
+ {
643
+ "entropy": 0.44672553464770315,
644
+ "epoch": 4.612691466083151,
645
+ "grad_norm": 0.27086368203163147,
646
+ "learning_rate": 3.627338623288235e-07,
647
+ "loss": 0.4741,
648
+ "mean_token_accuracy": 0.8914583876729012,
649
+ "num_tokens": 888414.0,
650
+ "step": 530
651
+ },
652
+ {
653
+ "entropy": 0.40851817429065707,
654
+ "epoch": 4.700218818380744,
655
+ "grad_norm": 0.31739744544029236,
656
+ "learning_rate": 2.6861237465589605e-07,
657
+ "loss": 0.4013,
658
+ "mean_token_accuracy": 0.8991223707795143,
659
+ "num_tokens": 905667.0,
660
+ "step": 540
661
+ },
662
+ {
663
+ "entropy": 0.41046958304941655,
664
+ "epoch": 4.787746170678337,
665
+ "grad_norm": 0.42413270473480225,
666
+ "learning_rate": 1.8827366782984913e-07,
667
+ "loss": 0.3903,
668
+ "mean_token_accuracy": 0.9042438387870788,
669
+ "num_tokens": 922133.0,
670
+ "step": 550
671
+ },
672
+ {
673
+ "epoch": 4.787746170678337,
674
+ "eval_entropy": 0.4343983823502505,
675
+ "eval_loss": 0.4322338402271271,
676
+ "eval_mean_token_accuracy": 0.8926174390463182,
677
+ "eval_num_tokens": 922133.0,
678
+ "eval_runtime": 10.4029,
679
+ "eval_samples_per_second": 7.786,
680
+ "eval_steps_per_second": 7.786,
681
+ "step": 550
682
+ },
683
+ {
684
+ "entropy": 0.40818401947617533,
685
+ "epoch": 4.87527352297593,
686
+ "grad_norm": 0.3421127200126648,
687
+ "learning_rate": 1.2195177272648129e-07,
688
+ "loss": 0.3905,
689
+ "mean_token_accuracy": 0.9041952416300774,
690
+ "num_tokens": 938442.0,
691
+ "step": 560
692
+ },
693
+ {
694
+ "entropy": 0.42612724639475347,
695
+ "epoch": 4.962800875273523,
696
+ "grad_norm": 0.3438023626804352,
697
+ "learning_rate": 6.983988851228473e-08,
698
+ "loss": 0.4149,
699
+ "mean_token_accuracy": 0.9018551856279373,
700
+ "num_tokens": 954845.0,
701
+ "step": 570
702
+ },
703
+ {
704
+ "entropy": 0.41250966730955485,
705
+ "epoch": 5.043763676148797,
706
+ "grad_norm": 0.27104368805885315,
707
+ "learning_rate": 3.2089819845111946e-08,
708
+ "loss": 0.394,
709
+ "mean_token_accuracy": 0.8978421961939013,
710
+ "num_tokens": 971145.0,
711
+ "step": 580
712
+ },
713
+ {
714
+ "entropy": 0.4053498174995184,
715
+ "epoch": 5.131291028446389,
716
+ "grad_norm": 0.2730412185192108,
717
+ "learning_rate": 8.811534659234899e-09,
718
+ "loss": 0.3956,
719
+ "mean_token_accuracy": 0.9008755519986152,
720
+ "num_tokens": 988466.0,
721
+ "step": 590
722
+ },
723
+ {
724
+ "entropy": 0.4064184818416834,
725
+ "epoch": 5.218818380743983,
726
+ "grad_norm": 0.2767721116542816,
727
+ "learning_rate": 7.284382296801618e-11,
728
+ "loss": 0.391,
729
+ "mean_token_accuracy": 0.9021652072668076,
730
+ "num_tokens": 1004947.0,
731
+ "step": 600
732
+ },
733
+ {
734
+ "epoch": 5.218818380743983,
735
+ "eval_entropy": 0.43397615371662895,
736
+ "eval_loss": 0.4320242702960968,
737
+ "eval_mean_token_accuracy": 0.8926537603507807,
738
+ "eval_num_tokens": 1004947.0,
739
+ "eval_runtime": 10.3967,
740
+ "eval_samples_per_second": 7.791,
741
+ "eval_steps_per_second": 7.791,
742
+ "step": 600
743
+ }
744
+ ],
745
+ "logging_steps": 10,
746
+ "max_steps": 600,
747
+ "num_input_tokens_seen": 0,
748
+ "num_train_epochs": 6,
749
+ "save_steps": 100,
750
+ "stateful_callbacks": {
751
+ "TrainerControl": {
752
+ "args": {
753
+ "should_epoch_stop": false,
754
+ "should_evaluate": false,
755
+ "should_log": false,
756
+ "should_save": true,
757
+ "should_training_stop": true
758
+ },
759
+ "attributes": {}
760
+ }
761
+ },
762
+ "total_flos": 4.617244215241114e+16,
763
+ "train_batch_size": 1,
764
+ "trial_name": null,
765
+ "trial_params": null
766
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7edbeaf20dd7f571b5dd1c54d9ace4f9b6299127cc7ba2afb14a6d51a4a79a4
3
+ size 17078136
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff