Hugging Face's logo

dipta007
/
GanitLLM-4B-SFT

Text Generation
Transformers
Safetensors
Bengali
English
qwen3
math
bengali
reasoning
sft
conversational
text-generation-inference
Model card Files
xet
Community
dipta007 commited on
Commit
101a237
·
verified ·
1 Parent(s): 5cf0c74

update files

Browse files
Files changed (25) hide show
  1. .gitattributes +1 -0
  2. added_tokens.json +28 -0
  3. args.json +383 -0
  4. chat_template.jinja +89 -0
  5. config.json +68 -0
  6. generation_config.json +13 -0
  7. global_step432/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +3 -0
  8. global_step432/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +3 -0
  9. global_step432/zero_pp_rank_0_mp_rank_00_model_states.pt +3 -0
  10. global_step432/zero_pp_rank_1_mp_rank_00_model_states.pt +3 -0
  11. latest +1 -0
  12. merges.txt +0 -0
  13. model-00001-of-00002.safetensors +3 -0
  14. model-00002-of-00002.safetensors +3 -0
  15. model.safetensors.index.json +406 -0
  16. rng_state_0.pth +3 -0
  17. rng_state_1.pth +3 -0
  18. scheduler.pt +3 -0
  19. special_tokens_map.json +31 -0
  20. tokenizer.json +3 -0
  21. tokenizer_config.json +239 -0
  22. trainer_state.json +3058 -0
  23. training_args.bin +3 -0
  24. vocab.json +0 -0
  25. zero_to_fp32.py +760 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
args.json ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "/scratch/kmahbub/GanitLLM_checkpoints/sft_v2/acl_full_Qwen3-4B/v6-20251017-010658",
3
+ "overwrite_output_dir": false,
4
+ "do_train": false,
5
+ "do_eval": false,
6
+ "do_predict": false,
7
+ "eval_strategy": "no",
8
+ "prediction_loss_only": false,
9
+ "per_device_train_batch_size": 16,
10
+ "per_device_eval_batch_size": 1,
11
+ "per_gpu_train_batch_size": null,
12
+ "per_gpu_eval_batch_size": null,
13
+ "gradient_accumulation_steps": 4,
14
+ "eval_accumulation_steps": null,
15
+ "eval_delay": 0,
16
+ "torch_empty_cache_steps": null,
17
+ "learning_rate": 1e-06,
18
+ "weight_decay": 0.1,
19
+ "adam_beta1": 0.9,
20
+ "adam_beta2": 0.95,
21
+ "adam_epsilon": 1e-08,
22
+ "max_grad_norm": 1.0,
23
+ "num_train_epochs": 50.0,
24
+ "max_steps": -1,
25
+ "lr_scheduler_type": "cosine_with_min_lr",
26
+ "lr_scheduler_kwargs": "{\"min_lr\": 1e-7}",
27
+ "warmup_ratio": 0.0,
28
+ "warmup_steps": 0,
29
+ "log_level": "passive",
30
+ "log_level_replica": "warning",
31
+ "log_on_each_node": true,
32
+ "logging_dir": "/scratch/kmahbub/GanitLLM_checkpoints/sft_v2/acl_full_Qwen3-4B/v6-20251017-010658/runs",
33
+ "logging_strategy": "steps",
34
+ "logging_first_step": true,
35
+ "logging_steps": 1,
36
+ "logging_nan_inf_filter": true,
37
+ "save_strategy": "epoch",
38
+ "save_steps": 1.0,
39
+ "save_total_limit": null,
40
+ "save_safetensors": true,
41
+ "save_on_each_node": false,
42
+ "save_only_model": false,
43
+ "restore_callback_states_from_checkpoint": false,
44
+ "no_cuda": false,
45
+ "use_cpu": false,
46
+ "use_mps_device": false,
47
+ "seed": 42,
48
+ "data_seed": 42,
49
+ "jit_mode_eval": false,
50
+ "bf16": true,
51
+ "fp16": false,
52
+ "fp16_opt_level": "O1",
53
+ "half_precision_backend": "auto",
54
+ "bf16_full_eval": false,
55
+ "fp16_full_eval": false,
56
+ "tf32": null,
57
+ "local_rank": 0,
58
+ "ddp_backend": null,
59
+ "tpu_num_cores": null,
60
+ "tpu_metrics_debug": false,
61
+ "debug": null,
62
+ "dataloader_drop_last": false,
63
+ "eval_steps": null,
64
+ "dataloader_num_workers": null,
65
+ "dataloader_prefetch_factor": null,
66
+ "past_index": -1,
67
+ "run_name": "acl_v2_sft_full_Qwen3-4B",
68
+ "disable_tqdm": null,
69
+ "remove_unused_columns": true,
70
+ "label_names": null,
71
+ "load_best_model_at_end": false,
72
+ "metric_for_best_model": "loss",
73
+ "greater_is_better": false,
74
+ "ignore_data_skip": false,
75
+ "fsdp": null,
76
+ "fsdp_min_num_params": 0,
77
+ "fsdp_config": null,
78
+ "fsdp_transformer_layer_cls_to_wrap": null,
79
+ "accelerator_config": {
80
+ "dispatch_batches": false
81
+ },
82
+ "parallelism_config": null,
83
+ "deepspeed": {
84
+ "fp16": {
85
+ "enabled": "auto",
86
+ "loss_scale": 0,
87
+ "loss_scale_window": 1000,
88
+ "initial_scale_power": 16,
89
+ "hysteresis": 2,
90
+ "min_loss_scale": 1
91
+ },
92
+ "bf16": {
93
+ "enabled": "auto"
94
+ },
95
+ "zero_optimization": {
96
+ "stage": 3,
97
+ "offload_optimizer": {
98
+ "device": "none",
99
+ "pin_memory": true
100
+ },
101
+ "offload_param": {
102
+ "device": "none",
103
+ "pin_memory": true
104
+ },
105
+ "overlap_comm": false,
106
+ "contiguous_gradients": true,
107
+ "sub_group_size": 1000000000.0,
108
+ "reduce_bucket_size": "auto",
109
+ "zero_quantized_weights": false,
110
+ "zero_quantized_gradients": false,
111
+ "stage3_prefetch_bucket_size": "auto",
112
+ "stage3_param_persistence_threshold": "auto",
113
+ "stage3_max_live_parameters": 1000000000.0,
114
+ "stage3_max_reuse_distance": 1000000000.0,
115
+ "stage3_gather_16bit_weights_on_model_save": true
116
+ },
117
+ "gradient_accumulation_steps": "auto",
118
+ "gradient_clipping": "auto",
119
+ "steps_per_print": 2000,
120
+ "train_batch_size": "auto",
121
+ "train_micro_batch_size_per_gpu": "auto",
122
+ "wall_clock_breakdown": false
123
+ },
124
+ "label_smoothing_factor": 0.0,
125
+ "optim": "adamw_torch_fused",
126
+ "optim_args": null,
127
+ "adafactor": false,
128
+ "group_by_length": false,
129
+ "length_column_name": "length",
130
+ "report_to": [
131
+ "wandb"
132
+ ],
133
+ "project": "huggingface",
134
+ "trackio_space_id": "trackio",
135
+ "ddp_find_unused_parameters": null,
136
+ "ddp_bucket_cap_mb": null,
137
+ "ddp_broadcast_buffers": null,
138
+ "dataloader_pin_memory": true,
139
+ "dataloader_persistent_workers": false,
140
+ "skip_memory_metrics": true,
141
+ "use_legacy_prediction_loop": false,
142
+ "push_to_hub": false,
143
+ "resume_from_checkpoint": null,
144
+ "hub_model_id": null,
145
+ "hub_strategy": "every_save",
146
+ "hub_token": null,
147
+ "hub_private_repo": null,
148
+ "hub_always_push": false,
149
+ "hub_revision": null,
150
+ "gradient_checkpointing": true,
151
+ "gradient_checkpointing_kwargs": null,
152
+ "include_inputs_for_metrics": false,
153
+ "include_for_metrics": [],
154
+ "eval_do_concat_batches": true,
155
+ "fp16_backend": "auto",
156
+ "push_to_hub_model_id": null,
157
+ "push_to_hub_organization": null,
158
+ "push_to_hub_token": null,
159
+ "mp_parameters": "",
160
+ "auto_find_batch_size": false,
161
+ "full_determinism": false,
162
+ "torchdynamo": null,
163
+ "ray_scope": "last",
164
+ "ddp_timeout": 18000000,
165
+ "torch_compile": false,
166
+ "torch_compile_backend": null,
167
+ "torch_compile_mode": null,
168
+ "include_tokens_per_second": false,
169
+ "include_num_input_tokens_seen": false,
170
+ "neftune_noise_alpha": null,
171
+ "optim_target_modules": null,
172
+ "batch_eval_metrics": false,
173
+ "eval_on_start": false,
174
+ "use_liger_kernel": true,
175
+ "liger_kernel_config": null,
176
+ "eval_use_gather_object": false,
177
+ "average_tokens_across_devices": true,
178
+ "sortish_sampler": false,
179
+ "predict_with_generate": false,
180
+ "generation_max_length": null,
181
+ "generation_num_beams": null,
182
+ "generation_config": null,
183
+ "tuner_backend": "peft",
184
+ "vit_gradient_checkpointing": null,
185
+ "router_aux_loss_coef": 0.0,
186
+ "enable_dft_loss": false,
187
+ "enable_channel_loss": false,
188
+ "check_model": true,
189
+ "acc_strategy": "token",
190
+ "train_dataloader_shuffle": true,
191
+ "max_epochs": null,
192
+ "aligner_lr": null,
193
+ "vit_lr": null,
194
+ "use_logits_to_keep": null,
195
+ "ds3_gather_for_generation": true,
196
+ "resume_only_model": false,
197
+ "optimizer": null,
198
+ "loss_type": null,
199
+ "metric": null,
200
+ "eval_use_evalscope": false,
201
+ "eval_dataset": [],
202
+ "eval_dataset_args": null,
203
+ "eval_limit": null,
204
+ "eval_generation_config": null,
205
+ "extra_eval_args": null,
206
+ "use_flash_ckpt": false,
207
+ "model": "Qwen/Qwen3-4B",
208
+ "model_type": "qwen3",
209
+ "model_revision": null,
210
+ "task_type": "causal_lm",
211
+ "torch_dtype": "bfloat16",
212
+ "attn_impl": "flash_attention_2",
213
+ "new_special_tokens": [],
214
+ "num_labels": null,
215
+ "problem_type": null,
216
+ "rope_scaling": null,
217
+ "device_map": null,
218
+ "max_memory": {},
219
+ "max_model_len": null,
220
+ "local_repo_path": null,
221
+ "init_strategy": null,
222
+ "template": "qwen3",
223
+ "system": null,
224
+ "max_length": 4096,
225
+ "truncation_strategy": "delete",
226
+ "max_pixels": null,
227
+ "agent_template": null,
228
+ "norm_bbox": null,
229
+ "use_chat_template": true,
230
+ "padding_free": true,
231
+ "padding_side": "right",
232
+ "loss_scale": "default",
233
+ "sequence_parallel_size": 1,
234
+ "response_prefix": null,
235
+ "template_backend": "swift",
236
+ "dataset": [
237
+ "data_v2/sft/thinking_train.jsonl"
238
+ ],
239
+ "val_dataset": [],
240
+ "split_dataset_ratio": 0.0,
241
+ "dataset_num_proc": 1,
242
+ "load_from_cache_file": false,
243
+ "dataset_shuffle": true,
244
+ "val_dataset_shuffle": false,
245
+ "streaming": false,
246
+ "interleave_prob": null,
247
+ "stopping_strategy": "first_exhausted",
248
+ "shuffle_buffer_size": 1000,
249
+ "download_mode": "reuse_dataset_if_exists",
250
+ "columns": {},
251
+ "strict": false,
252
+ "model_name": null,
253
+ "model_author": null,
254
+ "custom_dataset_info": [],
255
+ "quant_method": null,
256
+ "quant_bits": null,
257
+ "hqq_axis": null,
258
+ "bnb_4bit_compute_dtype": "bfloat16",
259
+ "bnb_4bit_quant_type": "nf4",
260
+ "bnb_4bit_use_double_quant": true,
261
+ "bnb_4bit_quant_storage": null,
262
+ "max_new_tokens": 64,
263
+ "temperature": 0.0,
264
+ "top_k": null,
265
+ "top_p": null,
266
+ "repetition_penalty": null,
267
+ "num_beams": 1,
268
+ "stream": false,
269
+ "stop_words": [],
270
+ "logprobs": false,
271
+ "top_logprobs": null,
272
+ "ckpt_dir": null,
273
+ "lora_modules": [],
274
+ "train_type": "full",
275
+ "adapters": [],
276
+ "external_plugins": [],
277
+ "model_kwargs": {},
278
+ "load_args": false,
279
+ "load_data_args": false,
280
+ "packing": true,
281
+ "packing_length": 4096,
282
+ "lazy_tokenize": false,
283
+ "cached_dataset": [],
284
+ "custom_register_path": [],
285
+ "use_hf": true,
286
+ "ignore_args_error": false,
287
+ "use_swift_lora": false,
288
+ "freeze_parameters": [],
289
+ "freeze_parameters_regex": null,
290
+ "freeze_parameters_ratio": 0.0,
291
+ "trainable_parameters": [],
292
+ "trainable_parameters_regex": null,
293
+ "freeze_llm": false,
294
+ "freeze_vit": true,
295
+ "freeze_aligner": true,
296
+ "target_modules": [
297
+ "all-linear"
298
+ ],
299
+ "target_regex": null,
300
+ "target_parameters": null,
301
+ "modules_to_save": [],
302
+ "lora_rank": 8,
303
+ "lora_alpha": 32,
304
+ "lora_dropout": 0.05,
305
+ "lora_bias": "none",
306
+ "lora_dtype": null,
307
+ "lorap_lr_ratio": null,
308
+ "use_rslora": false,
309
+ "use_dora": false,
310
+ "lora_ga_batch_size": 2,
311
+ "lora_ga_iters": 2,
312
+ "lora_ga_max_length": 1024,
313
+ "lora_ga_direction": "ArB2r",
314
+ "lora_ga_scale": "stable",
315
+ "lora_ga_stable_gamma": 16,
316
+ "init_weights": true,
317
+ "fourier_n_frequency": 2000,
318
+ "fourier_scaling": 300.0,
319
+ "boft_block_size": 4,
320
+ "boft_block_num": 0,
321
+ "boft_n_butterfly_factor": 1,
322
+ "boft_dropout": 0.0,
323
+ "vera_rank": 256,
324
+ "vera_projection_prng_key": 0,
325
+ "vera_dropout": 0.0,
326
+ "vera_d_initial": 0.1,
327
+ "adapter_act": "gelu",
328
+ "adapter_length": 128,
329
+ "use_galore": false,
330
+ "galore_target_modules": null,
331
+ "galore_rank": 128,
332
+ "galore_update_proj_gap": 50,
333
+ "galore_scale": 1.0,
334
+ "galore_proj_type": "std",
335
+ "galore_optim_per_parameter": false,
336
+ "galore_with_embedding": false,
337
+ "galore_quantization": false,
338
+ "galore_proj_quant": false,
339
+ "galore_proj_bits": 4,
340
+ "galore_proj_group_size": 256,
341
+ "galore_cos_threshold": 0.4,
342
+ "galore_gamma_proj": 2,
343
+ "galore_queue_size": 5,
344
+ "adalora_target_r": 8,
345
+ "adalora_init_r": 12,
346
+ "adalora_tinit": 0,
347
+ "adalora_tfinal": 0,
348
+ "adalora_deltaT": 1,
349
+ "adalora_beta1": 0.85,
350
+ "adalora_beta2": 0.85,
351
+ "adalora_orth_reg_weight": 0.5,
352
+ "llamapro_num_new_blocks": 4,
353
+ "llamapro_num_groups": null,
354
+ "lisa_activated_layers": 0,
355
+ "lisa_step_interval": 20,
356
+ "reft_layer_key": null,
357
+ "reft_layers": null,
358
+ "reft_rank": 4,
359
+ "reft_intervention_type": "LoreftIntervention",
360
+ "reft_args": null,
361
+ "swanlab_token": null,
362
+ "swanlab_project": null,
363
+ "swanlab_workspace": null,
364
+ "swanlab_exp_name": null,
365
+ "swanlab_lark_webhook_url": null,
366
+ "swanlab_lark_secret": null,
367
+ "swanlab_mode": "cloud",
368
+ "add_version": true,
369
+ "create_checkpoint_symlink": true,
370
+ "zero_hpz_partition_size": null,
371
+ "deepspeed_autotp_size": null,
372
+ "early_stop_interval": null,
373
+ "rank": 0,
374
+ "global_world_size": 2,
375
+ "local_world_size": 2,
376
+ "model_suffix": "Qwen3-4B",
377
+ "model_info": "ModelInfo(model_type='qwen3', model_dir='/users/kmahbub/.cache/huggingface/hub/models--Qwen--Qwen3-4B/snapshots/1cfa9a7208912126459214e8b04321603b3df60c', torch_dtype=torch.bfloat16, max_model_len=40960, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)",
378
+ "model_meta": "ModelMeta(model_type='qwen3', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-0.6B-Base', hf_model_id='Qwen/Qwen3-0.6B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-Base', hf_model_id='Qwen/Qwen3-1.7B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-Base', hf_model_id='Qwen/Qwen3-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-Base', hf_model_id='Qwen/Qwen3-8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-Base', hf_model_id='Qwen/Qwen3-14B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B', hf_model_id='Qwen/Qwen3-0.6B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B', hf_model_id='Qwen/Qwen3-1.7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B', hf_model_id='Qwen/Qwen3-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B', hf_model_id='Qwen/Qwen3-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B', hf_model_id='Qwen/Qwen3-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B', hf_model_id='Qwen/Qwen3-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B-FP8', hf_model_id='Qwen/Qwen3-0.6B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-FP8', hf_model_id='Qwen/Qwen3-1.7B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-FP8', hf_model_id='Qwen/Qwen3-4B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-FP8', hf_model_id='Qwen/Qwen3-8B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-FP8', hf_model_id='Qwen/Qwen3-14B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-FP8', hf_model_id='Qwen/Qwen3-32B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-AWQ', hf_model_id='Qwen/Qwen3-4B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-AWQ', hf_model_id='Qwen/Qwen3-8B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-AWQ', hf_model_id='Qwen/Qwen3-14B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-AWQ', hf_model_id='Qwen/Qwen3-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='swift/Qwen3-32B-AWQ', hf_model_id=None, model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen3', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f1811b63ba0>, model_arch=ModelKeys(arch_name='llama', embedding='model.embed_tokens', module_list='model.layers', lm_head='lm_head', q_proj='model.layers.{}.self_attn.q_proj', k_proj='model.layers.{}.self_attn.k_proj', v_proj='model.layers.{}.self_attn.v_proj', o_proj='model.layers.{}.self_attn.o_proj', attention='model.layers.{}.self_attn', mlp='model.layers.{}.mlp', down_proj='model.layers.{}.mlp.down_proj', qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None), architectures=['Qwen3ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.51'], tags=[])",
379
+ "model_dir": "/users/kmahbub/.cache/huggingface/hub/models--Qwen--Qwen3-4B/snapshots/1cfa9a7208912126459214e8b04321603b3df60c",
380
+ "hub": "<class 'swift.hub.hub.HFHub'>",
381
+ "evaluation_strategy": "no",
382
+ "training_args": "Seq2SeqTrainingArguments(output_dir='/scratch/kmahbub/GanitLLM_checkpoints/sft_v2/acl_full_Qwen3-4B/v6-20251017-010658', overwrite_output_dir=False, do_train=False, do_eval=False, do_predict=False, eval_strategy=<IntervalStrategy.NO: 'no'>, prediction_loss_only=False, per_device_train_batch_size=16, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=4, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-06, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=50.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE_WITH_MIN_LR: 'cosine_with_min_lr'>, lr_scheduler_kwargs={'min_lr': 1e-07}, warmup_ratio=0.0, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/scratch/kmahbub/GanitLLM_checkpoints/sft_v2/acl_full_Qwen3-4B/v6-20251017-010658/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=1.0, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=None, dataloader_num_workers=1, dataloader_prefetch_factor=10, past_index=-1, run_name='acl_v2_sft_full_Qwen3-4B', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['wandb'], project='huggingface', trackio_space_id='trackio', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=True, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, chord_sft_dataset=[], chord_sft_per_device_train_batch_size=None, chord_enable_phi_function=False, chord_mu_warmup_steps=None, chord_mu_decay_steps=None, chord_mu_peak=None, chord_mu_valley=None, train_type='full', local_repo_path=None, galore_config=None, padding_side='right', padding_free=True, task_type='causal_lm')"
383
+ }
chat_template.jinja ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0].role == 'system' %}
4
+ {{- messages[0].content + '\n\n' }}
5
+ {%- endif %}
6
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
7
+ {%- for tool in tools %}
8
+ {{- "\n" }}
9
+ {{- tool | tojson }}
10
+ {%- endfor %}
11
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
12
+ {%- else %}
13
+ {%- if messages[0].role == 'system' %}
14
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
18
+ {%- for message in messages[::-1] %}
19
+ {%- set index = (messages|length - 1) - loop.index0 %}
20
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
21
+ {%- set ns.multi_step_tool = false %}
22
+ {%- set ns.last_query_index = index %}
23
+ {%- endif %}
24
+ {%- endfor %}
25
+ {%- for message in messages %}
26
+ {%- if message.content is string %}
27
+ {%- set content = message.content %}
28
+ {%- else %}
29
+ {%- set content = '' %}
30
+ {%- endif %}
31
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
32
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
33
+ {%- elif message.role == "assistant" %}
34
+ {%- set reasoning_content = '' %}
35
+ {%- if message.reasoning_content is string %}
36
+ {%- set reasoning_content = message.reasoning_content %}
37
+ {%- else %}
38
+ {%- if '</think>' in content %}
39
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
40
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
41
+ {%- endif %}
42
+ {%- endif %}
43
+ {%- if loop.index0 > ns.last_query_index %}
44
+ {%- if loop.last or (not loop.last and reasoning_content) %}
45
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
46
+ {%- else %}
47
+ {{- '<|im_start|>' + message.role + '\n' + content }}
48
+ {%- endif %}
49
+ {%- else %}
50
+ {{- '<|im_start|>' + message.role + '\n' + content }}
51
+ {%- endif %}
52
+ {%- if message.tool_calls %}
53
+ {%- for tool_call in message.tool_calls %}
54
+ {%- if (loop.first and content) or (not loop.first) %}
55
+ {{- '\n' }}
56
+ {%- endif %}
57
+ {%- if tool_call.function %}
58
+ {%- set tool_call = tool_call.function %}
59
+ {%- endif %}
60
+ {{- '<tool_call>\n{"name": "' }}
61
+ {{- tool_call.name }}
62
+ {{- '", "arguments": ' }}
63
+ {%- if tool_call.arguments is string %}
64
+ {{- tool_call.arguments }}
65
+ {%- else %}
66
+ {{- tool_call.arguments | tojson }}
67
+ {%- endif %}
68
+ {{- '}\n</tool_call>' }}
69
+ {%- endfor %}
70
+ {%- endif %}
71
+ {{- '<|im_end|>\n' }}
72
+ {%- elif message.role == "tool" %}
73
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
74
+ {{- '<|im_start|>user' }}
75
+ {%- endif %}
76
+ {{- '\n<tool_response>\n' }}
77
+ {{- content }}
78
+ {{- '\n</tool_response>' }}
79
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
80
+ {{- '<|im_end|>\n' }}
81
+ {%- endif %}
82
+ {%- endif %}
83
+ {%- endfor %}
84
+ {%- if add_generation_prompt %}
85
+ {{- '<|im_start|>assistant\n' }}
86
+ {%- if enable_thinking is defined and enable_thinking is false %}
87
+ {{- '<think>\n\n</think>\n\n' }}
88
+ {%- endif %}
89
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3ForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": 151645,
9
+ "head_dim": 128,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2560,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 9728,
14
+ "layer_types": [
15
+ "full_attention",
16
+ "full_attention",
17
+ "full_attention",
18
+ "full_attention",
19
+ "full_attention",
20
+ "full_attention",
21
+ "full_attention",
22
+ "full_attention",
23
+ "full_attention",
24
+ "full_attention",
25
+ "full_attention",
26
+ "full_attention",
27
+ "full_attention",
28
+ "full_attention",
29
+ "full_attention",
30
+ "full_attention",
31
+ "full_attention",
32
+ "full_attention",
33
+ "full_attention",
34
+ "full_attention",
35
+ "full_attention",
36
+ "full_attention",
37
+ "full_attention",
38
+ "full_attention",
39
+ "full_attention",
40
+ "full_attention",
41
+ "full_attention",
42
+ "full_attention",
43
+ "full_attention",
44
+ "full_attention",
45
+ "full_attention",
46
+ "full_attention",
47
+ "full_attention",
48
+ "full_attention",
49
+ "full_attention",
50
+ "full_attention"
51
+ ],
52
+ "max_position_embeddings": 40960,
53
+ "max_window_layers": 36,
54
+ "model_type": "qwen3",
55
+ "num_attention_heads": 32,
56
+ "num_hidden_layers": 36,
57
+ "num_key_value_heads": 8,
58
+ "pad_token_id": 151643,
59
+ "rms_norm_eps": 1e-06,
60
+ "rope_scaling": null,
61
+ "rope_theta": 1000000,
62
+ "sliding_window": null,
63
+ "tie_word_embeddings": true,
64
+ "transformers_version": "4.57.1",
65
+ "use_cache": false,
66
+ "use_sliding_window": false,
67
+ "vocab_size": 151936
68
+ }
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 151643,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 151645,
6
+ 151643
7
+ ],
8
+ "pad_token_id": 151643,
9
+ "temperature": 0.6,
10
+ "top_k": 20,
11
+ "top_p": 0.95,
12
+ "transformers_version": "4.57.1"
13
+ }
global_step432/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6bd26c386a47439d6b35f6a1c7e275e40f26f3146ca678b468b347a8d020bb31
3
+ size 24134815827
global_step432/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d9bc83fb4cada1bcdac2ae54847c7d4211d6a5ff3ca0f70c7277e8ff75c55c4
3
+ size 24134815827
global_step432/zero_pp_rank_0_mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d09ae05646a3b10ea9cba5bb306558948230c481d9d78cf882528a607acafe3
3
+ size 202767
global_step432/zero_pp_rank_1_mp_rank_00_model_states.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c98367ae76c5773282c9fbd21710435d185ae4f0eafe5373b65d23e2213cd78
3
+ size 202703
latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step432
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc1794335df77163fab667c8087bcf6e30c8377a5177900b48b2c14a7e24e87b
3
+ size 4967215360
model-00002-of-00002.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec9cf129eb6e57f7adcc385e4bde018109b664cffdb21162cac02e34aecbf7f8
3
+ size 3077766632
model.safetensors.index.json ADDED
@@ -0,0 +1,406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_parameters": 196096,
4
+ "total_size": 8044936192
5
+ },
6
+ "weight_map": {
7
+ "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
9
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
10
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
11
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
13
+ "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
14
+ "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
15
+ "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
16
+ "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
17
+ "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
18
+ "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
19
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
20
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
21
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
22
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
23
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
24
+ "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
25
+ "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
26
+ "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
27
+ "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
28
+ "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
29
+ "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
30
+ "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
31
+ "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
32
+ "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
33
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
34
+ "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
35
+ "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
36
+ "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
37
+ "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
38
+ "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
39
+ "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
40
+ "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
41
+ "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
42
+ "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
43
+ "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
44
+ "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
45
+ "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
46
+ "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
47
+ "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
48
+ "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
49
+ "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
50
+ "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
51
+ "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
52
+ "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
53
+ "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
54
+ "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
55
+ "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
56
+ "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
57
+ "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
58
+ "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
59
+ "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
60
+ "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
61
+ "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
62
+ "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
63
+ "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
64
+ "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
65
+ "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
66
+ "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
67
+ "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
68
+ "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
69
+ "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
70
+ "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
71
+ "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
72
+ "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
73
+ "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
74
+ "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
75
+ "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
76
+ "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
77
+ "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
78
+ "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
79
+ "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
80
+ "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
81
+ "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
82
+ "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
83
+ "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
84
+ "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
85
+ "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
86
+ "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
87
+ "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
88
+ "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
89
+ "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
90
+ "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
91
+ "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
92
+ "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
93
+ "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
94
+ "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
95
+ "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
96
+ "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
97
+ "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
98
+ "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
99
+ "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
100
+ "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
101
+ "model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
102
+ "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
103
+ "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
104
+ "model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
105
+ "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
106
+ "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
107
+ "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
108
+ "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
109
+ "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
110
+ "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
111
+ "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
112
+ "model.layers.17.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
113
+ "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
114
+ "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
115
+ "model.layers.17.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
116
+ "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
117
+ "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
118
+ "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
119
+ "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
120
+ "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
121
+ "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
122
+ "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
123
+ "model.layers.18.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
124
+ "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
125
+ "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
126
+ "model.layers.18.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
127
+ "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
128
+ "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
129
+ "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
130
+ "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
131
+ "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
132
+ "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
133
+ "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
134
+ "model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
135
+ "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
136
+ "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
137
+ "model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
138
+ "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
139
+ "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
140
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
141
+ "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
142
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
143
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
144
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
145
+ "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
146
+ "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
147
+ "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
148
+ "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
149
+ "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
150
+ "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
151
+ "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
152
+ "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
153
+ "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
154
+ "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
155
+ "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
156
+ "model.layers.20.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
157
+ "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
158
+ "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
159
+ "model.layers.20.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
160
+ "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
161
+ "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
162
+ "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
163
+ "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
164
+ "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
165
+ "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
166
+ "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
167
+ "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
168
+ "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
169
+ "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
170
+ "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
171
+ "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
172
+ "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
173
+ "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
174
+ "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
175
+ "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
176
+ "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
177
+ "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
178
+ "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
179
+ "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
180
+ "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
181
+ "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
182
+ "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
183
+ "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
184
+ "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
185
+ "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
186
+ "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
187
+ "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
188
+ "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
189
+ "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
190
+ "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
191
+ "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
192
+ "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
193
+ "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
194
+ "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
195
+ "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
196
+ "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
197
+ "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
198
+ "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
199
+ "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
200
+ "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
201
+ "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
202
+ "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
203
+ "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
204
+ "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
205
+ "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
206
+ "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
207
+ "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
208
+ "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
209
+ "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
210
+ "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
211
+ "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
212
+ "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
213
+ "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
214
+ "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
215
+ "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
216
+ "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
217
+ "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
218
+ "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
219
+ "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
220
+ "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
221
+ "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
222
+ "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
223
+ "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
224
+ "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
225
+ "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
226
+ "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
227
+ "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
228
+ "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
229
+ "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
230
+ "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
231
+ "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
232
+ "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
233
+ "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
234
+ "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
235
+ "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
236
+ "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
237
+ "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
238
+ "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
239
+ "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
240
+ "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
241
+ "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
242
+ "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
243
+ "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
244
+ "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
245
+ "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
246
+ "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
247
+ "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
248
+ "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
249
+ "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
250
+ "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
251
+ "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
252
+ "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
253
+ "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
254
+ "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
255
+ "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
256
+ "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
257
+ "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
258
+ "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
259
+ "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
260
+ "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
261
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
262
+ "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
263
+ "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
264
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
265
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
266
+ "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
267
+ "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
268
+ "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
269
+ "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
270
+ "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
271
+ "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
272
+ "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
273
+ "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
274
+ "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
275
+ "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
276
+ "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
277
+ "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
278
+ "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
279
+ "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
280
+ "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
281
+ "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
282
+ "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
283
+ "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
284
+ "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
285
+ "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
286
+ "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
287
+ "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
288
+ "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
289
+ "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
290
+ "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
291
+ "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
292
+ "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
293
+ "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
294
+ "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
295
+ "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
296
+ "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
297
+ "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
298
+ "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
299
+ "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
300
+ "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
301
+ "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
302
+ "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
303
+ "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
304
+ "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
305
+ "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
306
+ "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
307
+ "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
308
+ "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
309
+ "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
310
+ "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
311
+ "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
312
+ "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
313
+ "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
314
+ "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
315
+ "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
316
+ "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
317
+ "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
318
+ "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
319
+ "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
320
+ "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
321
+ "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
322
+ "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
323
+ "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
324
+ "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
325
+ "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
326
+ "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
327
+ "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
328
+ "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
329
+ "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
330
+ "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
331
+ "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
332
+ "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
333
+ "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
334
+ "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
335
+ "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
336
+ "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
337
+ "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
338
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
339
+ "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
340
+ "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
341
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
342
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
343
+ "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
344
+ "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
345
+ "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
346
+ "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
347
+ "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
348
+ "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
349
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
350
+ "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
351
+ "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
352
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
353
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
354
+ "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
355
+ "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
356
+ "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
357
+ "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
358
+ "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
359
+ "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
360
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
361
+ "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
362
+ "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
363
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
364
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
365
+ "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
366
+ "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
367
+ "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
368
+ "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
369
+ "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
370
+ "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
371
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
372
+ "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
373
+ "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
374
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
375
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
376
+ "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
377
+ "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
378
+ "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
379
+ "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
380
+ "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
381
+ "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
382
+ "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
383
+ "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
384
+ "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
385
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
386
+ "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
387
+ "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
388
+ "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
389
+ "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
390
+ "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
391
+ "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
392
+ "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
393
+ "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
394
+ "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
395
+ "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
396
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
397
+ "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
398
+ "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
399
+ "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
400
+ "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
401
+ "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
402
+ "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
403
+ "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
404
+ "model.norm.weight": "model-00002-of-00002.safetensors"
405
+ }
406
+ }
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcbc62deef6ee75bb1869849c898fdb966db205961838be0c24d47ab98b8522a
3
+ size 14917
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36ba7e21c57a16ca3c595f45d116b9c706070e7aa5ad129f1405f946addc6a49
3
+ size 14917
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbd5ba874ba5e14c4e43b9bbae39e6deda25ff9f61bb6bf27187c7f6818d6c9a
3
+ size 1465
special_tokens_map.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|object_ref_start|>",
6
+ "<|object_ref_end|>",
7
+ "<|box_start|>",
8
+ "<|box_end|>",
9
+ "<|quad_start|>",
10
+ "<|quad_end|>",
11
+ "<|vision_start|>",
12
+ "<|vision_end|>",
13
+ "<|vision_pad|>",
14
+ "<|image_pad|>",
15
+ "<|video_pad|>"
16
+ ],
17
+ "eos_token": {
18
+ "content": "<|im_end|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<|endoftext|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ }
31
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
3
+ size 11422654
tokenizer_config.json ADDED
@@ -0,0 +1,239 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "151643": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "151644": {
14
+ "content": "<|im_start|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "151645": {
22
+ "content": "<|im_end|>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "151646": {
30
+ "content": "<|object_ref_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "151647": {
38
+ "content": "<|object_ref_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "151648": {
46
+ "content": "<|box_start|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "151649": {
54
+ "content": "<|box_end|>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "151650": {
62
+ "content": "<|quad_start|>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "151651": {
70
+ "content": "<|quad_end|>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "151652": {
78
+ "content": "<|vision_start|>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ },
85
+ "151653": {
86
+ "content": "<|vision_end|>",
87
+ "lstrip": false,
88
+ "normalized": false,
89
+ "rstrip": false,
90
+ "single_word": false,
91
+ "special": true
92
+ },
93
+ "151654": {
94
+ "content": "<|vision_pad|>",
95
+ "lstrip": false,
96
+ "normalized": false,
97
+ "rstrip": false,
98
+ "single_word": false,
99
+ "special": true
100
+ },
101
+ "151655": {
102
+ "content": "<|image_pad|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false,
107
+ "special": true
108
+ },
109
+ "151656": {
110
+ "content": "<|video_pad|>",
111
+ "lstrip": false,
112
+ "normalized": false,
113
+ "rstrip": false,
114
+ "single_word": false,
115
+ "special": true
116
+ },
117
+ "151657": {
118
+ "content": "<tool_call>",
119
+ "lstrip": false,
120
+ "normalized": false,
121
+ "rstrip": false,
122
+ "single_word": false,
123
+ "special": false
124
+ },
125
+ "151658": {
126
+ "content": "</tool_call>",
127
+ "lstrip": false,
128
+ "normalized": false,
129
+ "rstrip": false,
130
+ "single_word": false,
131
+ "special": false
132
+ },
133
+ "151659": {
134
+ "content": "<|fim_prefix|>",
135
+ "lstrip": false,
136
+ "normalized": false,
137
+ "rstrip": false,
138
+ "single_word": false,
139
+ "special": false
140
+ },
141
+ "151660": {
142
+ "content": "<|fim_middle|>",
143
+ "lstrip": false,
144
+ "normalized": false,
145
+ "rstrip": false,
146
+ "single_word": false,
147
+ "special": false
148
+ },
149
+ "151661": {
150
+ "content": "<|fim_suffix|>",
151
+ "lstrip": false,
152
+ "normalized": false,
153
+ "rstrip": false,
154
+ "single_word": false,
155
+ "special": false
156
+ },
157
+ "151662": {
158
+ "content": "<|fim_pad|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false,
163
+ "special": false
164
+ },
165
+ "151663": {
166
+ "content": "<|repo_name|>",
167
+ "lstrip": false,
168
+ "normalized": false,
169
+ "rstrip": false,
170
+ "single_word": false,
171
+ "special": false
172
+ },
173
+ "151664": {
174
+ "content": "<|file_sep|>",
175
+ "lstrip": false,
176
+ "normalized": false,
177
+ "rstrip": false,
178
+ "single_word": false,
179
+ "special": false
180
+ },
181
+ "151665": {
182
+ "content": "<tool_response>",
183
+ "lstrip": false,
184
+ "normalized": false,
185
+ "rstrip": false,
186
+ "single_word": false,
187
+ "special": false
188
+ },
189
+ "151666": {
190
+ "content": "</tool_response>",
191
+ "lstrip": false,
192
+ "normalized": false,
193
+ "rstrip": false,
194
+ "single_word": false,
195
+ "special": false
196
+ },
197
+ "151667": {
198
+ "content": "<think>",
199
+ "lstrip": false,
200
+ "normalized": false,
201
+ "rstrip": false,
202
+ "single_word": false,
203
+ "special": false
204
+ },
205
+ "151668": {
206
+ "content": "</think>",
207
+ "lstrip": false,
208
+ "normalized": false,
209
+ "rstrip": false,
210
+ "single_word": false,
211
+ "special": false
212
+ }
213
+ },
214
+ "additional_special_tokens": [
215
+ "<|im_start|>",
216
+ "<|im_end|>",
217
+ "<|object_ref_start|>",
218
+ "<|object_ref_end|>",
219
+ "<|box_start|>",
220
+ "<|box_end|>",
221
+ "<|quad_start|>",
222
+ "<|quad_end|>",
223
+ "<|vision_start|>",
224
+ "<|vision_end|>",
225
+ "<|vision_pad|>",
226
+ "<|image_pad|>",
227
+ "<|video_pad|>"
228
+ ],
229
+ "bos_token": null,
230
+ "clean_up_tokenization_spaces": false,
231
+ "eos_token": "<|im_end|>",
232
+ "errors": "replace",
233
+ "extra_special_tokens": {},
234
+ "model_max_length": 131072,
235
+ "pad_token": "<|endoftext|>",
236
+ "split_special_tokens": false,
237
+ "tokenizer_class": "Qwen2Tokenizer",
238
+ "unk_token": null
239
+ }
trainer_state.json ADDED
@@ -0,0 +1,3058 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 12.0,
6
+ "eval_steps": 500,
7
+ "global_step": 432,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.028169014084507043,
14
+ "grad_norm": 5.418299028662257,
15
+ "learning_rate": 9.999993146109795e-07,
16
+ "loss": 0.5132051110267639,
17
+ "step": 1
18
+ },
19
+ {
20
+ "epoch": 0.056338028169014086,
21
+ "grad_norm": 5.165609347936129,
22
+ "learning_rate": 9.999972584460056e-07,
23
+ "loss": 0.5073634386062622,
24
+ "step": 2
25
+ },
26
+ {
27
+ "epoch": 0.08450704225352113,
28
+ "grad_norm": 5.195437990428209,
29
+ "learning_rate": 9.99993831511342e-07,
30
+ "loss": 0.5153955221176147,
31
+ "step": 3
32
+ },
33
+ {
34
+ "epoch": 0.11267605633802817,
35
+ "grad_norm": 4.79583419513962,
36
+ "learning_rate": 9.999890338174275e-07,
37
+ "loss": 0.4989941716194153,
38
+ "step": 4
39
+ },
40
+ {
41
+ "epoch": 0.14084507042253522,
42
+ "grad_norm": 4.207848876215671,
43
+ "learning_rate": 9.99982865378877e-07,
44
+ "loss": 0.48423337936401367,
45
+ "step": 5
46
+ },
47
+ {
48
+ "epoch": 0.16901408450704225,
49
+ "grad_norm": 4.218583800715356,
50
+ "learning_rate": 9.999753262144804e-07,
51
+ "loss": 0.48089924454689026,
52
+ "step": 6
53
+ },
54
+ {
55
+ "epoch": 0.19718309859154928,
56
+ "grad_norm": 4.005100353285046,
57
+ "learning_rate": 9.999664163472034e-07,
58
+ "loss": 0.47420695424079895,
59
+ "step": 7
60
+ },
61
+ {
62
+ "epoch": 0.22535211267605634,
63
+ "grad_norm": 3.929557609614918,
64
+ "learning_rate": 9.999561358041868e-07,
65
+ "loss": 0.47918927669525146,
66
+ "step": 8
67
+ },
68
+ {
69
+ "epoch": 0.2535211267605634,
70
+ "grad_norm": 2.694796145734545,
71
+ "learning_rate": 9.99944484616747e-07,
72
+ "loss": 0.4395635426044464,
73
+ "step": 9
74
+ },
75
+ {
76
+ "epoch": 0.28169014084507044,
77
+ "grad_norm": 2.427520188062667,
78
+ "learning_rate": 9.99931462820376e-07,
79
+ "loss": 0.42557471990585327,
80
+ "step": 10
81
+ },
82
+ {
83
+ "epoch": 0.30985915492957744,
84
+ "grad_norm": 2.4361246746804692,
85
+ "learning_rate": 9.999170704547398e-07,
86
+ "loss": 0.43526768684387207,
87
+ "step": 11
88
+ },
89
+ {
90
+ "epoch": 0.3380281690140845,
91
+ "grad_norm": 2.394762841017111,
92
+ "learning_rate": 9.999013075636804e-07,
93
+ "loss": 0.4402061104774475,
94
+ "step": 12
95
+ },
96
+ {
97
+ "epoch": 0.36619718309859156,
98
+ "grad_norm": 2.083652020594123,
99
+ "learning_rate": 9.998841741952141e-07,
100
+ "loss": 0.4102749824523926,
101
+ "step": 13
102
+ },
103
+ {
104
+ "epoch": 0.39436619718309857,
105
+ "grad_norm": 2.0892711360401948,
106
+ "learning_rate": 9.998656704015323e-07,
107
+ "loss": 0.42078256607055664,
108
+ "step": 14
109
+ },
110
+ {
111
+ "epoch": 0.4225352112676056,
112
+ "grad_norm": 2.0344088244674343,
113
+ "learning_rate": 9.998457962390008e-07,
114
+ "loss": 0.4181898832321167,
115
+ "step": 15
116
+ },
117
+ {
118
+ "epoch": 0.4507042253521127,
119
+ "grad_norm": 1.940959597352366,
120
+ "learning_rate": 9.998245517681593e-07,
121
+ "loss": 0.40955501794815063,
122
+ "step": 16
123
+ },
124
+ {
125
+ "epoch": 0.4788732394366197,
126
+ "grad_norm": 0.8816622805671499,
127
+ "learning_rate": 9.998019370537227e-07,
128
+ "loss": 0.3903525471687317,
129
+ "step": 17
130
+ },
131
+ {
132
+ "epoch": 0.5070422535211268,
133
+ "grad_norm": 0.8501184697349302,
134
+ "learning_rate": 9.997779521645791e-07,
135
+ "loss": 0.3796430230140686,
136
+ "step": 18
137
+ },
138
+ {
139
+ "epoch": 0.5352112676056338,
140
+ "grad_norm": 0.8116182529970104,
141
+ "learning_rate": 9.997525971737909e-07,
142
+ "loss": 0.3822035789489746,
143
+ "step": 19
144
+ },
145
+ {
146
+ "epoch": 0.5633802816901409,
147
+ "grad_norm": 0.7873864669427912,
148
+ "learning_rate": 9.997258721585931e-07,
149
+ "loss": 0.3768383860588074,
150
+ "step": 20
151
+ },
152
+ {
153
+ "epoch": 0.5915492957746479,
154
+ "grad_norm": 0.7538308754999873,
155
+ "learning_rate": 9.99697777200395e-07,
156
+ "loss": 0.3780366778373718,
157
+ "step": 21
158
+ },
159
+ {
160
+ "epoch": 0.6197183098591549,
161
+ "grad_norm": 0.746877326907616,
162
+ "learning_rate": 9.996683123847795e-07,
163
+ "loss": 0.37998318672180176,
164
+ "step": 22
165
+ },
166
+ {
167
+ "epoch": 0.647887323943662,
168
+ "grad_norm": 0.7186953728538551,
169
+ "learning_rate": 9.996374778015007e-07,
170
+ "loss": 0.37321770191192627,
171
+ "step": 23
172
+ },
173
+ {
174
+ "epoch": 0.676056338028169,
175
+ "grad_norm": 0.6944848844555613,
176
+ "learning_rate": 9.996052735444862e-07,
177
+ "loss": 0.3735676109790802,
178
+ "step": 24
179
+ },
180
+ {
181
+ "epoch": 0.704225352112676,
182
+ "grad_norm": 0.6362193589652109,
183
+ "learning_rate": 9.99571699711836e-07,
184
+ "loss": 0.37480324506759644,
185
+ "step": 25
186
+ },
187
+ {
188
+ "epoch": 0.7323943661971831,
189
+ "grad_norm": 0.6333241594644416,
190
+ "learning_rate": 9.995367564058216e-07,
191
+ "loss": 0.36993861198425293,
192
+ "step": 26
193
+ },
194
+ {
195
+ "epoch": 0.7605633802816901,
196
+ "grad_norm": 0.6098149107884999,
197
+ "learning_rate": 9.995004437328865e-07,
198
+ "loss": 0.3650243282318115,
199
+ "step": 27
200
+ },
201
+ {
202
+ "epoch": 0.7887323943661971,
203
+ "grad_norm": 0.5944375419192803,
204
+ "learning_rate": 9.994627618036452e-07,
205
+ "loss": 0.3905714750289917,
206
+ "step": 28
207
+ },
208
+ {
209
+ "epoch": 0.8169014084507042,
210
+ "grad_norm": 0.5736619217852436,
211
+ "learning_rate": 9.994237107328838e-07,
212
+ "loss": 0.3627358675003052,
213
+ "step": 29
214
+ },
215
+ {
216
+ "epoch": 0.8450704225352113,
217
+ "grad_norm": 0.5886929426905549,
218
+ "learning_rate": 9.993832906395582e-07,
219
+ "loss": 0.3565104603767395,
220
+ "step": 30
221
+ },
222
+ {
223
+ "epoch": 0.8732394366197183,
224
+ "grad_norm": 0.5353744882375503,
225
+ "learning_rate": 9.993415016467952e-07,
226
+ "loss": 0.36376649141311646,
227
+ "step": 31
228
+ },
229
+ {
230
+ "epoch": 0.9014084507042254,
231
+ "grad_norm": 0.49225367207832443,
232
+ "learning_rate": 9.992983438818915e-07,
233
+ "loss": 0.369179368019104,
234
+ "step": 32
235
+ },
236
+ {
237
+ "epoch": 0.9295774647887324,
238
+ "grad_norm": 0.4556896703645998,
239
+ "learning_rate": 9.992538174763127e-07,
240
+ "loss": 0.3526361882686615,
241
+ "step": 33
242
+ },
243
+ {
244
+ "epoch": 0.9577464788732394,
245
+ "grad_norm": 0.5188804849989508,
246
+ "learning_rate": 9.992079225656944e-07,
247
+ "loss": 0.3492780327796936,
248
+ "step": 34
249
+ },
250
+ {
251
+ "epoch": 0.9859154929577465,
252
+ "grad_norm": 0.5109563747610114,
253
+ "learning_rate": 9.9916065928984e-07,
254
+ "loss": 0.3500766158103943,
255
+ "step": 35
256
+ },
257
+ {
258
+ "epoch": 1.0,
259
+ "grad_norm": 0.5562934346388212,
260
+ "learning_rate": 9.991120277927223e-07,
261
+ "loss": 0.37105804681777954,
262
+ "step": 36
263
+ },
264
+ {
265
+ "epoch": 1.028169014084507,
266
+ "grad_norm": 0.5221736259155545,
267
+ "learning_rate": 9.990620282224806e-07,
268
+ "loss": 0.3426528871059418,
269
+ "step": 37
270
+ },
271
+ {
272
+ "epoch": 1.056338028169014,
273
+ "grad_norm": 0.5312140458936209,
274
+ "learning_rate": 9.990106607314225e-07,
275
+ "loss": 0.3402440547943115,
276
+ "step": 38
277
+ },
278
+ {
279
+ "epoch": 1.084507042253521,
280
+ "grad_norm": 0.4925804810345089,
281
+ "learning_rate": 9.989579254760224e-07,
282
+ "loss": 0.3478923439979553,
283
+ "step": 39
284
+ },
285
+ {
286
+ "epoch": 1.1126760563380282,
287
+ "grad_norm": 0.4713456639902427,
288
+ "learning_rate": 9.989038226169207e-07,
289
+ "loss": 0.3395651578903198,
290
+ "step": 40
291
+ },
292
+ {
293
+ "epoch": 1.1408450704225352,
294
+ "grad_norm": 0.40869144140431884,
295
+ "learning_rate": 9.988483523189248e-07,
296
+ "loss": 0.33732888102531433,
297
+ "step": 41
298
+ },
299
+ {
300
+ "epoch": 1.1690140845070423,
301
+ "grad_norm": 0.4315316017587138,
302
+ "learning_rate": 9.98791514751006e-07,
303
+ "loss": 0.32492154836654663,
304
+ "step": 42
305
+ },
306
+ {
307
+ "epoch": 1.1971830985915493,
308
+ "grad_norm": 0.37481864132677223,
309
+ "learning_rate": 9.98733310086302e-07,
310
+ "loss": 0.34133073687553406,
311
+ "step": 43
312
+ },
313
+ {
314
+ "epoch": 1.2253521126760563,
315
+ "grad_norm": 0.3574253218229588,
316
+ "learning_rate": 9.98673738502114e-07,
317
+ "loss": 0.3353017568588257,
318
+ "step": 44
319
+ },
320
+ {
321
+ "epoch": 1.2535211267605635,
322
+ "grad_norm": 0.3440064791263243,
323
+ "learning_rate": 9.986128001799076e-07,
324
+ "loss": 0.3392435312271118,
325
+ "step": 45
326
+ },
327
+ {
328
+ "epoch": 1.2816901408450705,
329
+ "grad_norm": 0.340263808482957,
330
+ "learning_rate": 9.985504953053113e-07,
331
+ "loss": 0.3362383544445038,
332
+ "step": 46
333
+ },
334
+ {
335
+ "epoch": 1.3098591549295775,
336
+ "grad_norm": 0.33745919836931715,
337
+ "learning_rate": 9.984868240681164e-07,
338
+ "loss": 0.3242315649986267,
339
+ "step": 47
340
+ },
341
+ {
342
+ "epoch": 1.3380281690140845,
343
+ "grad_norm": 0.3289741626977548,
344
+ "learning_rate": 9.98421786662277e-07,
345
+ "loss": 0.3304663896560669,
346
+ "step": 48
347
+ },
348
+ {
349
+ "epoch": 1.3661971830985915,
350
+ "grad_norm": 0.31460186215251207,
351
+ "learning_rate": 9.983553832859078e-07,
352
+ "loss": 0.32084885239601135,
353
+ "step": 49
354
+ },
355
+ {
356
+ "epoch": 1.3943661971830985,
357
+ "grad_norm": 0.3245154924065007,
358
+ "learning_rate": 9.982876141412855e-07,
359
+ "loss": 0.3394209146499634,
360
+ "step": 50
361
+ },
362
+ {
363
+ "epoch": 1.4225352112676055,
364
+ "grad_norm": 0.31250544820074627,
365
+ "learning_rate": 9.982184794348462e-07,
366
+ "loss": 0.32516294717788696,
367
+ "step": 51
368
+ },
369
+ {
370
+ "epoch": 1.4507042253521127,
371
+ "grad_norm": 0.30500960409480415,
372
+ "learning_rate": 9.981479793771866e-07,
373
+ "loss": 0.33079296350479126,
374
+ "step": 52
375
+ },
376
+ {
377
+ "epoch": 1.4788732394366197,
378
+ "grad_norm": 0.7536289161405646,
379
+ "learning_rate": 9.98076114183062e-07,
380
+ "loss": 0.32204341888427734,
381
+ "step": 53
382
+ },
383
+ {
384
+ "epoch": 1.5070422535211268,
385
+ "grad_norm": 0.2960511511830655,
386
+ "learning_rate": 9.98002884071386e-07,
387
+ "loss": 0.3231901526451111,
388
+ "step": 54
389
+ },
390
+ {
391
+ "epoch": 1.5352112676056338,
392
+ "grad_norm": 0.3232856362024084,
393
+ "learning_rate": 9.979282892652304e-07,
394
+ "loss": 0.33578696846961975,
395
+ "step": 55
396
+ },
397
+ {
398
+ "epoch": 1.563380281690141,
399
+ "grad_norm": 0.2944501744644002,
400
+ "learning_rate": 9.97852329991824e-07,
401
+ "loss": 0.32318681478500366,
402
+ "step": 56
403
+ },
404
+ {
405
+ "epoch": 1.591549295774648,
406
+ "grad_norm": 0.2929838195254813,
407
+ "learning_rate": 9.977750064825519e-07,
408
+ "loss": 0.339860737323761,
409
+ "step": 57
410
+ },
411
+ {
412
+ "epoch": 1.619718309859155,
413
+ "grad_norm": 0.3205944314956805,
414
+ "learning_rate": 9.976963189729547e-07,
415
+ "loss": 0.32377177476882935,
416
+ "step": 58
417
+ },
418
+ {
419
+ "epoch": 1.647887323943662,
420
+ "grad_norm": 0.29345488313940016,
421
+ "learning_rate": 9.976162677027284e-07,
422
+ "loss": 0.3301998972892761,
423
+ "step": 59
424
+ },
425
+ {
426
+ "epoch": 1.676056338028169,
427
+ "grad_norm": 0.28471478486927526,
428
+ "learning_rate": 9.975348529157229e-07,
429
+ "loss": 0.3226792812347412,
430
+ "step": 60
431
+ },
432
+ {
433
+ "epoch": 1.704225352112676,
434
+ "grad_norm": 0.2658831538134308,
435
+ "learning_rate": 9.974520748599421e-07,
436
+ "loss": 0.3216060996055603,
437
+ "step": 61
438
+ },
439
+ {
440
+ "epoch": 1.732394366197183,
441
+ "grad_norm": 0.26772727111733485,
442
+ "learning_rate": 9.973679337875418e-07,
443
+ "loss": 0.3112826347351074,
444
+ "step": 62
445
+ },
446
+ {
447
+ "epoch": 1.76056338028169,
448
+ "grad_norm": 0.27607375529137707,
449
+ "learning_rate": 9.972824299548309e-07,
450
+ "loss": 0.32318398356437683,
451
+ "step": 63
452
+ },
453
+ {
454
+ "epoch": 1.788732394366197,
455
+ "grad_norm": 0.276525714664102,
456
+ "learning_rate": 9.971955636222684e-07,
457
+ "loss": 0.3215365707874298,
458
+ "step": 64
459
+ },
460
+ {
461
+ "epoch": 1.8169014084507042,
462
+ "grad_norm": 0.26621366711174155,
463
+ "learning_rate": 9.971073350544644e-07,
464
+ "loss": 0.3165343403816223,
465
+ "step": 65
466
+ },
467
+ {
468
+ "epoch": 1.8450704225352113,
469
+ "grad_norm": 0.28106696236531414,
470
+ "learning_rate": 9.970177445201783e-07,
471
+ "loss": 0.32160013914108276,
472
+ "step": 66
473
+ },
474
+ {
475
+ "epoch": 1.8732394366197183,
476
+ "grad_norm": 0.2647958460920452,
477
+ "learning_rate": 9.969267922923188e-07,
478
+ "loss": 0.3175761103630066,
479
+ "step": 67
480
+ },
481
+ {
482
+ "epoch": 1.9014084507042255,
483
+ "grad_norm": 0.3462438167974764,
484
+ "learning_rate": 9.968344786479415e-07,
485
+ "loss": 0.31158342957496643,
486
+ "step": 68
487
+ },
488
+ {
489
+ "epoch": 1.9295774647887325,
490
+ "grad_norm": 0.2631517138191484,
491
+ "learning_rate": 9.967408038682505e-07,
492
+ "loss": 0.3191946744918823,
493
+ "step": 69
494
+ },
495
+ {
496
+ "epoch": 1.9577464788732395,
497
+ "grad_norm": 0.3059517169103558,
498
+ "learning_rate": 9.96645768238595e-07,
499
+ "loss": 0.31686434149742126,
500
+ "step": 70
501
+ },
502
+ {
503
+ "epoch": 1.9859154929577465,
504
+ "grad_norm": 0.2747560759134169,
505
+ "learning_rate": 9.965493720484698e-07,
506
+ "loss": 0.32002022862434387,
507
+ "step": 71
508
+ },
509
+ {
510
+ "epoch": 2.0,
511
+ "grad_norm": 0.3436945075098925,
512
+ "learning_rate": 9.964516155915151e-07,
513
+ "loss": 0.31206002831459045,
514
+ "step": 72
515
+ },
516
+ {
517
+ "epoch": 2.028169014084507,
518
+ "grad_norm": 0.24759643540050102,
519
+ "learning_rate": 9.963524991655133e-07,
520
+ "loss": 0.3164379596710205,
521
+ "step": 73
522
+ },
523
+ {
524
+ "epoch": 2.056338028169014,
525
+ "grad_norm": 0.2434100698887695,
526
+ "learning_rate": 9.962520230723906e-07,
527
+ "loss": 0.31132200360298157,
528
+ "step": 74
529
+ },
530
+ {
531
+ "epoch": 2.084507042253521,
532
+ "grad_norm": 0.27767056283467373,
533
+ "learning_rate": 9.961501876182148e-07,
534
+ "loss": 0.3142748475074768,
535
+ "step": 75
536
+ },
537
+ {
538
+ "epoch": 2.112676056338028,
539
+ "grad_norm": 0.2383770994918481,
540
+ "learning_rate": 9.960469931131936e-07,
541
+ "loss": 0.3118811249732971,
542
+ "step": 76
543
+ },
544
+ {
545
+ "epoch": 2.140845070422535,
546
+ "grad_norm": 0.2521646078141003,
547
+ "learning_rate": 9.959424398716763e-07,
548
+ "loss": 0.3206384778022766,
549
+ "step": 77
550
+ },
551
+ {
552
+ "epoch": 2.169014084507042,
553
+ "grad_norm": 0.24500926882339497,
554
+ "learning_rate": 9.958365282121496e-07,
555
+ "loss": 0.3028615117073059,
556
+ "step": 78
557
+ },
558
+ {
559
+ "epoch": 2.1971830985915495,
560
+ "grad_norm": 0.24030528199838755,
561
+ "learning_rate": 9.95729258457239e-07,
562
+ "loss": 0.2991243600845337,
563
+ "step": 79
564
+ },
565
+ {
566
+ "epoch": 2.2253521126760565,
567
+ "grad_norm": 0.24540461017636186,
568
+ "learning_rate": 9.956206309337066e-07,
569
+ "loss": 0.3028961420059204,
570
+ "step": 80
571
+ },
572
+ {
573
+ "epoch": 2.2535211267605635,
574
+ "grad_norm": 0.2466946916886877,
575
+ "learning_rate": 9.955106459724508e-07,
576
+ "loss": 0.3096856474876404,
577
+ "step": 81
578
+ },
579
+ {
580
+ "epoch": 2.2816901408450705,
581
+ "grad_norm": 0.2755173295195889,
582
+ "learning_rate": 9.953993039085048e-07,
583
+ "loss": 0.30638784170150757,
584
+ "step": 82
585
+ },
586
+ {
587
+ "epoch": 2.3098591549295775,
588
+ "grad_norm": 0.2514653938379217,
589
+ "learning_rate": 9.952866050810363e-07,
590
+ "loss": 0.3001154661178589,
591
+ "step": 83
592
+ },
593
+ {
594
+ "epoch": 2.3380281690140845,
595
+ "grad_norm": 0.2387732228915944,
596
+ "learning_rate": 9.951725498333448e-07,
597
+ "loss": 0.3049381673336029,
598
+ "step": 84
599
+ },
600
+ {
601
+ "epoch": 2.3661971830985915,
602
+ "grad_norm": 0.23263762732735094,
603
+ "learning_rate": 9.950571385128625e-07,
604
+ "loss": 0.2974574565887451,
605
+ "step": 85
606
+ },
607
+ {
608
+ "epoch": 2.3943661971830985,
609
+ "grad_norm": 0.23061479782330688,
610
+ "learning_rate": 9.949403714711526e-07,
611
+ "loss": 0.2910880148410797,
612
+ "step": 86
613
+ },
614
+ {
615
+ "epoch": 2.4225352112676055,
616
+ "grad_norm": 0.23023752633245553,
617
+ "learning_rate": 9.948222490639075e-07,
618
+ "loss": 0.2984412610530853,
619
+ "step": 87
620
+ },
621
+ {
622
+ "epoch": 2.4507042253521125,
623
+ "grad_norm": 0.2556642237814582,
624
+ "learning_rate": 9.947027716509488e-07,
625
+ "loss": 0.29366716742515564,
626
+ "step": 88
627
+ },
628
+ {
629
+ "epoch": 2.4788732394366195,
630
+ "grad_norm": 0.23091894334348845,
631
+ "learning_rate": 9.94581939596225e-07,
632
+ "loss": 0.3076009750366211,
633
+ "step": 89
634
+ },
635
+ {
636
+ "epoch": 2.507042253521127,
637
+ "grad_norm": 0.22827065781305456,
638
+ "learning_rate": 9.944597532678119e-07,
639
+ "loss": 0.30507659912109375,
640
+ "step": 90
641
+ },
642
+ {
643
+ "epoch": 2.535211267605634,
644
+ "grad_norm": 0.23495783877634624,
645
+ "learning_rate": 9.943362130379101e-07,
646
+ "loss": 0.30069300532341003,
647
+ "step": 91
648
+ },
649
+ {
650
+ "epoch": 2.563380281690141,
651
+ "grad_norm": 0.2648421541967289,
652
+ "learning_rate": 9.942113192828444e-07,
653
+ "loss": 0.3237234652042389,
654
+ "step": 92
655
+ },
656
+ {
657
+ "epoch": 2.591549295774648,
658
+ "grad_norm": 0.2376008941714217,
659
+ "learning_rate": 9.940850723830632e-07,
660
+ "loss": 0.30564817786216736,
661
+ "step": 93
662
+ },
663
+ {
664
+ "epoch": 2.619718309859155,
665
+ "grad_norm": 0.3594923854547647,
666
+ "learning_rate": 9.939574727231362e-07,
667
+ "loss": 0.2874234914779663,
668
+ "step": 94
669
+ },
670
+ {
671
+ "epoch": 2.647887323943662,
672
+ "grad_norm": 0.22882707129774643,
673
+ "learning_rate": 9.93828520691754e-07,
674
+ "loss": 0.3145613670349121,
675
+ "step": 95
676
+ },
677
+ {
678
+ "epoch": 2.676056338028169,
679
+ "grad_norm": 0.22736499094759352,
680
+ "learning_rate": 9.93698216681727e-07,
681
+ "loss": 0.30298489332199097,
682
+ "step": 96
683
+ },
684
+ {
685
+ "epoch": 2.704225352112676,
686
+ "grad_norm": 0.2560977265944999,
687
+ "learning_rate": 9.93566561089984e-07,
688
+ "loss": 0.30830079317092896,
689
+ "step": 97
690
+ },
691
+ {
692
+ "epoch": 2.732394366197183,
693
+ "grad_norm": 0.23579803469255398,
694
+ "learning_rate": 9.934335543175705e-07,
695
+ "loss": 0.2954327166080475,
696
+ "step": 98
697
+ },
698
+ {
699
+ "epoch": 2.76056338028169,
700
+ "grad_norm": 0.24145604440100388,
701
+ "learning_rate": 9.932991967696482e-07,
702
+ "loss": 0.3123829960823059,
703
+ "step": 99
704
+ },
705
+ {
706
+ "epoch": 2.788732394366197,
707
+ "grad_norm": 0.23937111018917692,
708
+ "learning_rate": 9.931634888554935e-07,
709
+ "loss": 0.30813831090927124,
710
+ "step": 100
711
+ },
712
+ {
713
+ "epoch": 2.816901408450704,
714
+ "grad_norm": 0.23102456481544026,
715
+ "learning_rate": 9.930264309884964e-07,
716
+ "loss": 0.29939332604408264,
717
+ "step": 101
718
+ },
719
+ {
720
+ "epoch": 2.845070422535211,
721
+ "grad_norm": 0.2388325654329505,
722
+ "learning_rate": 9.928880235861588e-07,
723
+ "loss": 0.3060687184333801,
724
+ "step": 102
725
+ },
726
+ {
727
+ "epoch": 2.873239436619718,
728
+ "grad_norm": 0.2692772021272968,
729
+ "learning_rate": 9.927482670700936e-07,
730
+ "loss": 0.3026995360851288,
731
+ "step": 103
732
+ },
733
+ {
734
+ "epoch": 2.9014084507042255,
735
+ "grad_norm": 0.2195252664324123,
736
+ "learning_rate": 9.926071618660237e-07,
737
+ "loss": 0.29397717118263245,
738
+ "step": 104
739
+ },
740
+ {
741
+ "epoch": 2.9295774647887325,
742
+ "grad_norm": 0.24295446606786492,
743
+ "learning_rate": 9.924647084037797e-07,
744
+ "loss": 0.30009227991104126,
745
+ "step": 105
746
+ },
747
+ {
748
+ "epoch": 2.9577464788732395,
749
+ "grad_norm": 0.2313070418168474,
750
+ "learning_rate": 9.923209071172994e-07,
751
+ "loss": 0.2927643656730652,
752
+ "step": 106
753
+ },
754
+ {
755
+ "epoch": 2.9859154929577465,
756
+ "grad_norm": 0.22879166840885404,
757
+ "learning_rate": 9.921757584446268e-07,
758
+ "loss": 0.2916420102119446,
759
+ "step": 107
760
+ },
761
+ {
762
+ "epoch": 3.0,
763
+ "grad_norm": 0.3008754833697553,
764
+ "learning_rate": 9.9202926282791e-07,
765
+ "loss": 0.28270626068115234,
766
+ "step": 108
767
+ },
768
+ {
769
+ "epoch": 3.028169014084507,
770
+ "grad_norm": 0.23568699154112785,
771
+ "learning_rate": 9.918814207133997e-07,
772
+ "loss": 0.28633755445480347,
773
+ "step": 109
774
+ },
775
+ {
776
+ "epoch": 3.056338028169014,
777
+ "grad_norm": 0.23303560144939867,
778
+ "learning_rate": 9.917322325514487e-07,
779
+ "loss": 0.30074095726013184,
780
+ "step": 110
781
+ },
782
+ {
783
+ "epoch": 3.084507042253521,
784
+ "grad_norm": 0.2316049153802507,
785
+ "learning_rate": 9.915816987965102e-07,
786
+ "loss": 0.2938191294670105,
787
+ "step": 111
788
+ },
789
+ {
790
+ "epoch": 3.112676056338028,
791
+ "grad_norm": 0.24039981484305117,
792
+ "learning_rate": 9.91429819907136e-07,
793
+ "loss": 0.3015148639678955,
794
+ "step": 112
795
+ },
796
+ {
797
+ "epoch": 3.140845070422535,
798
+ "grad_norm": 0.21607780251617179,
799
+ "learning_rate": 9.912765963459756e-07,
800
+ "loss": 0.2841828167438507,
801
+ "step": 113
802
+ },
803
+ {
804
+ "epoch": 3.169014084507042,
805
+ "grad_norm": 0.24291861075598384,
806
+ "learning_rate": 9.911220285797748e-07,
807
+ "loss": 0.2961827218532562,
808
+ "step": 114
809
+ },
810
+ {
811
+ "epoch": 3.1971830985915495,
812
+ "grad_norm": 0.260079633277976,
813
+ "learning_rate": 9.909661170793733e-07,
814
+ "loss": 0.2853129506111145,
815
+ "step": 115
816
+ },
817
+ {
818
+ "epoch": 3.2253521126760565,
819
+ "grad_norm": 0.22330476823859097,
820
+ "learning_rate": 9.908088623197048e-07,
821
+ "loss": 0.30180466175079346,
822
+ "step": 116
823
+ },
824
+ {
825
+ "epoch": 3.2535211267605635,
826
+ "grad_norm": 0.23280629394443647,
827
+ "learning_rate": 9.906502647797945e-07,
828
+ "loss": 0.30052781105041504,
829
+ "step": 117
830
+ },
831
+ {
832
+ "epoch": 3.2816901408450705,
833
+ "grad_norm": 0.21797810381540847,
834
+ "learning_rate": 9.904903249427582e-07,
835
+ "loss": 0.2998088598251343,
836
+ "step": 118
837
+ },
838
+ {
839
+ "epoch": 3.3098591549295775,
840
+ "grad_norm": 0.2122579387275699,
841
+ "learning_rate": 9.903290432958003e-07,
842
+ "loss": 0.2904450297355652,
843
+ "step": 119
844
+ },
845
+ {
846
+ "epoch": 3.3380281690140845,
847
+ "grad_norm": 0.2306597760843549,
848
+ "learning_rate": 9.901664203302124e-07,
849
+ "loss": 0.2772594690322876,
850
+ "step": 120
851
+ },
852
+ {
853
+ "epoch": 3.3661971830985915,
854
+ "grad_norm": 0.3251131669552687,
855
+ "learning_rate": 9.900024565413727e-07,
856
+ "loss": 0.2782062292098999,
857
+ "step": 121
858
+ },
859
+ {
860
+ "epoch": 3.3943661971830985,
861
+ "grad_norm": 0.21492053378836454,
862
+ "learning_rate": 9.89837152428743e-07,
863
+ "loss": 0.2862273156642914,
864
+ "step": 122
865
+ },
866
+ {
867
+ "epoch": 3.4225352112676055,
868
+ "grad_norm": 0.21940072001551747,
869
+ "learning_rate": 9.896705084958687e-07,
870
+ "loss": 0.2992798686027527,
871
+ "step": 123
872
+ },
873
+ {
874
+ "epoch": 3.4507042253521125,
875
+ "grad_norm": 0.22907980661604213,
876
+ "learning_rate": 9.895025252503755e-07,
877
+ "loss": 0.27001839876174927,
878
+ "step": 124
879
+ },
880
+ {
881
+ "epoch": 3.4788732394366195,
882
+ "grad_norm": 0.22323548107714866,
883
+ "learning_rate": 9.8933320320397e-07,
884
+ "loss": 0.2867587208747864,
885
+ "step": 125
886
+ },
887
+ {
888
+ "epoch": 3.507042253521127,
889
+ "grad_norm": 0.2133180650070634,
890
+ "learning_rate": 9.891625428724364e-07,
891
+ "loss": 0.2889852821826935,
892
+ "step": 126
893
+ },
894
+ {
895
+ "epoch": 3.535211267605634,
896
+ "grad_norm": 0.22234390437617468,
897
+ "learning_rate": 9.889905447756355e-07,
898
+ "loss": 0.2830744981765747,
899
+ "step": 127
900
+ },
901
+ {
902
+ "epoch": 3.563380281690141,
903
+ "grad_norm": 0.22998700794578517,
904
+ "learning_rate": 9.888172094375033e-07,
905
+ "loss": 0.29788801074028015,
906
+ "step": 128
907
+ },
908
+ {
909
+ "epoch": 3.591549295774648,
910
+ "grad_norm": 0.4625877582933462,
911
+ "learning_rate": 9.886425373860496e-07,
912
+ "loss": 0.29489147663116455,
913
+ "step": 129
914
+ },
915
+ {
916
+ "epoch": 3.619718309859155,
917
+ "grad_norm": 0.22935899493352271,
918
+ "learning_rate": 9.88466529153356e-07,
919
+ "loss": 0.2970463037490845,
920
+ "step": 130
921
+ },
922
+ {
923
+ "epoch": 3.647887323943662,
924
+ "grad_norm": 0.24274389064678412,
925
+ "learning_rate": 9.882891852755732e-07,
926
+ "loss": 0.2851516008377075,
927
+ "step": 131
928
+ },
929
+ {
930
+ "epoch": 3.676056338028169,
931
+ "grad_norm": 0.22141629938400506,
932
+ "learning_rate": 9.881105062929221e-07,
933
+ "loss": 0.2885037660598755,
934
+ "step": 132
935
+ },
936
+ {
937
+ "epoch": 3.704225352112676,
938
+ "grad_norm": 0.22752563556013145,
939
+ "learning_rate": 9.879304927496896e-07,
940
+ "loss": 0.28612369298934937,
941
+ "step": 133
942
+ },
943
+ {
944
+ "epoch": 3.732394366197183,
945
+ "grad_norm": 0.21651661785773998,
946
+ "learning_rate": 9.877491451942284e-07,
947
+ "loss": 0.2883659601211548,
948
+ "step": 134
949
+ },
950
+ {
951
+ "epoch": 3.76056338028169,
952
+ "grad_norm": 0.23438766576379866,
953
+ "learning_rate": 9.875664641789543e-07,
954
+ "loss": 0.27838006615638733,
955
+ "step": 135
956
+ },
957
+ {
958
+ "epoch": 3.788732394366197,
959
+ "grad_norm": 0.2253998931695203,
960
+ "learning_rate": 9.873824502603459e-07,
961
+ "loss": 0.2768632471561432,
962
+ "step": 136
963
+ },
964
+ {
965
+ "epoch": 3.816901408450704,
966
+ "grad_norm": 0.2177655119766866,
967
+ "learning_rate": 9.871971039989407e-07,
968
+ "loss": 0.2836754322052002,
969
+ "step": 137
970
+ },
971
+ {
972
+ "epoch": 3.845070422535211,
973
+ "grad_norm": 0.22595230341732617,
974
+ "learning_rate": 9.870104259593362e-07,
975
+ "loss": 0.28235411643981934,
976
+ "step": 138
977
+ },
978
+ {
979
+ "epoch": 3.873239436619718,
980
+ "grad_norm": 0.2482489166005438,
981
+ "learning_rate": 9.86822416710186e-07,
982
+ "loss": 0.29619723558425903,
983
+ "step": 139
984
+ },
985
+ {
986
+ "epoch": 3.9014084507042255,
987
+ "grad_norm": 0.2183147080685166,
988
+ "learning_rate": 9.866330768241983e-07,
989
+ "loss": 0.2837688624858856,
990
+ "step": 140
991
+ },
992
+ {
993
+ "epoch": 3.9295774647887325,
994
+ "grad_norm": 0.23756245135618922,
995
+ "learning_rate": 9.86442406878136e-07,
996
+ "loss": 0.29686689376831055,
997
+ "step": 141
998
+ },
999
+ {
1000
+ "epoch": 3.9577464788732395,
1001
+ "grad_norm": 0.21910640405689585,
1002
+ "learning_rate": 9.862504074528126e-07,
1003
+ "loss": 0.3005070984363556,
1004
+ "step": 142
1005
+ },
1006
+ {
1007
+ "epoch": 3.9859154929577465,
1008
+ "grad_norm": 0.2231139496297488,
1009
+ "learning_rate": 9.860570791330911e-07,
1010
+ "loss": 0.28214913606643677,
1011
+ "step": 143
1012
+ },
1013
+ {
1014
+ "epoch": 4.0,
1015
+ "grad_norm": 0.327745139722082,
1016
+ "learning_rate": 9.85862422507884e-07,
1017
+ "loss": 0.2614859938621521,
1018
+ "step": 144
1019
+ },
1020
+ {
1021
+ "epoch": 4.028169014084507,
1022
+ "grad_norm": 0.21956180832878694,
1023
+ "learning_rate": 9.856664381701483e-07,
1024
+ "loss": 0.2801395654678345,
1025
+ "step": 145
1026
+ },
1027
+ {
1028
+ "epoch": 4.056338028169014,
1029
+ "grad_norm": 0.21981533940846626,
1030
+ "learning_rate": 9.854691267168871e-07,
1031
+ "loss": 0.2805081307888031,
1032
+ "step": 146
1033
+ },
1034
+ {
1035
+ "epoch": 4.084507042253521,
1036
+ "grad_norm": 0.22352024368863976,
1037
+ "learning_rate": 9.852704887491445e-07,
1038
+ "loss": 0.2810959815979004,
1039
+ "step": 147
1040
+ },
1041
+ {
1042
+ "epoch": 4.112676056338028,
1043
+ "grad_norm": 0.21378340480206676,
1044
+ "learning_rate": 9.850705248720068e-07,
1045
+ "loss": 0.2902711033821106,
1046
+ "step": 148
1047
+ },
1048
+ {
1049
+ "epoch": 4.140845070422535,
1050
+ "grad_norm": 0.6576619030457451,
1051
+ "learning_rate": 9.848692356945981e-07,
1052
+ "loss": 0.2759314775466919,
1053
+ "step": 149
1054
+ },
1055
+ {
1056
+ "epoch": 4.169014084507042,
1057
+ "grad_norm": 0.22012344359855593,
1058
+ "learning_rate": 9.846666218300807e-07,
1059
+ "loss": 0.268562912940979,
1060
+ "step": 150
1061
+ },
1062
+ {
1063
+ "epoch": 4.197183098591549,
1064
+ "grad_norm": 0.21540756213399562,
1065
+ "learning_rate": 9.844626838956513e-07,
1066
+ "loss": 0.29484277963638306,
1067
+ "step": 151
1068
+ },
1069
+ {
1070
+ "epoch": 4.225352112676056,
1071
+ "grad_norm": 0.23087516265191377,
1072
+ "learning_rate": 9.8425742251254e-07,
1073
+ "loss": 0.2736097574234009,
1074
+ "step": 152
1075
+ },
1076
+ {
1077
+ "epoch": 4.253521126760563,
1078
+ "grad_norm": 0.22858752073312144,
1079
+ "learning_rate": 9.84050838306009e-07,
1080
+ "loss": 0.2795490622520447,
1081
+ "step": 153
1082
+ },
1083
+ {
1084
+ "epoch": 4.28169014084507,
1085
+ "grad_norm": 0.2504089824174044,
1086
+ "learning_rate": 9.838429319053495e-07,
1087
+ "loss": 0.27981656789779663,
1088
+ "step": 154
1089
+ },
1090
+ {
1091
+ "epoch": 4.309859154929577,
1092
+ "grad_norm": 0.2144903706614124,
1093
+ "learning_rate": 9.836337039438803e-07,
1094
+ "loss": 0.2772422134876251,
1095
+ "step": 155
1096
+ },
1097
+ {
1098
+ "epoch": 4.338028169014084,
1099
+ "grad_norm": 0.23097398374379755,
1100
+ "learning_rate": 9.83423155058946e-07,
1101
+ "loss": 0.2819685637950897,
1102
+ "step": 156
1103
+ },
1104
+ {
1105
+ "epoch": 4.366197183098592,
1106
+ "grad_norm": 0.24080056507624897,
1107
+ "learning_rate": 9.832112858919155e-07,
1108
+ "loss": 0.28476956486701965,
1109
+ "step": 157
1110
+ },
1111
+ {
1112
+ "epoch": 4.394366197183099,
1113
+ "grad_norm": 0.22574168772954528,
1114
+ "learning_rate": 9.829980970881784e-07,
1115
+ "loss": 0.2726949155330658,
1116
+ "step": 158
1117
+ },
1118
+ {
1119
+ "epoch": 4.422535211267606,
1120
+ "grad_norm": 0.6972322742026779,
1121
+ "learning_rate": 9.82783589297145e-07,
1122
+ "loss": 0.2827332615852356,
1123
+ "step": 159
1124
+ },
1125
+ {
1126
+ "epoch": 4.450704225352113,
1127
+ "grad_norm": 0.2391641337184146,
1128
+ "learning_rate": 9.825677631722435e-07,
1129
+ "loss": 0.28474992513656616,
1130
+ "step": 160
1131
+ },
1132
+ {
1133
+ "epoch": 4.47887323943662,
1134
+ "grad_norm": 0.221898655707498,
1135
+ "learning_rate": 9.823506193709174e-07,
1136
+ "loss": 0.28681856393814087,
1137
+ "step": 161
1138
+ },
1139
+ {
1140
+ "epoch": 4.507042253521127,
1141
+ "grad_norm": 0.22336161749059524,
1142
+ "learning_rate": 9.821321585546243e-07,
1143
+ "loss": 0.2809918522834778,
1144
+ "step": 162
1145
+ },
1146
+ {
1147
+ "epoch": 4.535211267605634,
1148
+ "grad_norm": 0.21973730928374757,
1149
+ "learning_rate": 9.81912381388834e-07,
1150
+ "loss": 0.265026330947876,
1151
+ "step": 163
1152
+ },
1153
+ {
1154
+ "epoch": 4.563380281690141,
1155
+ "grad_norm": 0.25521582293188844,
1156
+ "learning_rate": 9.816912885430258e-07,
1157
+ "loss": 0.27740782499313354,
1158
+ "step": 164
1159
+ },
1160
+ {
1161
+ "epoch": 4.591549295774648,
1162
+ "grad_norm": 0.22348230986657833,
1163
+ "learning_rate": 9.814688806906868e-07,
1164
+ "loss": 0.2830485701560974,
1165
+ "step": 165
1166
+ },
1167
+ {
1168
+ "epoch": 4.619718309859155,
1169
+ "grad_norm": 0.23949143954055024,
1170
+ "learning_rate": 9.812451585093098e-07,
1171
+ "loss": 0.2732582092285156,
1172
+ "step": 166
1173
+ },
1174
+ {
1175
+ "epoch": 4.647887323943662,
1176
+ "grad_norm": 0.24011757347977605,
1177
+ "learning_rate": 9.810201226803917e-07,
1178
+ "loss": 0.27790567278862,
1179
+ "step": 167
1180
+ },
1181
+ {
1182
+ "epoch": 4.676056338028169,
1183
+ "grad_norm": 0.2208241856254554,
1184
+ "learning_rate": 9.807937738894303e-07,
1185
+ "loss": 0.27999186515808105,
1186
+ "step": 168
1187
+ },
1188
+ {
1189
+ "epoch": 4.704225352112676,
1190
+ "grad_norm": 0.22207182950383192,
1191
+ "learning_rate": 9.805661128259235e-07,
1192
+ "loss": 0.2758759558200836,
1193
+ "step": 169
1194
+ },
1195
+ {
1196
+ "epoch": 4.732394366197183,
1197
+ "grad_norm": 0.2223130884819787,
1198
+ "learning_rate": 9.80337140183366e-07,
1199
+ "loss": 0.2783896327018738,
1200
+ "step": 170
1201
+ },
1202
+ {
1203
+ "epoch": 4.76056338028169,
1204
+ "grad_norm": 0.2251473945003646,
1205
+ "learning_rate": 9.801068566592483e-07,
1206
+ "loss": 0.2750943601131439,
1207
+ "step": 171
1208
+ },
1209
+ {
1210
+ "epoch": 4.788732394366197,
1211
+ "grad_norm": 0.20904181797718158,
1212
+ "learning_rate": 9.798752629550546e-07,
1213
+ "loss": 0.2790430784225464,
1214
+ "step": 172
1215
+ },
1216
+ {
1217
+ "epoch": 4.816901408450704,
1218
+ "grad_norm": 0.22738399964996078,
1219
+ "learning_rate": 9.796423597762588e-07,
1220
+ "loss": 0.2752009630203247,
1221
+ "step": 173
1222
+ },
1223
+ {
1224
+ "epoch": 4.845070422535211,
1225
+ "grad_norm": 0.3424040844187128,
1226
+ "learning_rate": 9.794081478323245e-07,
1227
+ "loss": 0.2811102271080017,
1228
+ "step": 174
1229
+ },
1230
+ {
1231
+ "epoch": 4.873239436619718,
1232
+ "grad_norm": 0.22370787445632553,
1233
+ "learning_rate": 9.791726278367021e-07,
1234
+ "loss": 0.27801671624183655,
1235
+ "step": 175
1236
+ },
1237
+ {
1238
+ "epoch": 4.901408450704225,
1239
+ "grad_norm": 0.244597442698197,
1240
+ "learning_rate": 9.78935800506826e-07,
1241
+ "loss": 0.2749514579772949,
1242
+ "step": 176
1243
+ },
1244
+ {
1245
+ "epoch": 4.929577464788732,
1246
+ "grad_norm": 0.22603846349643272,
1247
+ "learning_rate": 9.786976665641138e-07,
1248
+ "loss": 0.2660600543022156,
1249
+ "step": 177
1250
+ },
1251
+ {
1252
+ "epoch": 4.957746478873239,
1253
+ "grad_norm": 0.22615957713600704,
1254
+ "learning_rate": 9.784582267339622e-07,
1255
+ "loss": 0.28135621547698975,
1256
+ "step": 178
1257
+ },
1258
+ {
1259
+ "epoch": 4.985915492957746,
1260
+ "grad_norm": 0.22802110120296137,
1261
+ "learning_rate": 9.78217481745747e-07,
1262
+ "loss": 0.27813419699668884,
1263
+ "step": 179
1264
+ },
1265
+ {
1266
+ "epoch": 5.0,
1267
+ "grad_norm": 0.31867363735197585,
1268
+ "learning_rate": 9.779754323328192e-07,
1269
+ "loss": 0.28184953331947327,
1270
+ "step": 180
1271
+ },
1272
+ {
1273
+ "epoch": 5.028169014084507,
1274
+ "grad_norm": 0.23359190602929383,
1275
+ "learning_rate": 9.777320792325025e-07,
1276
+ "loss": 0.2600752115249634,
1277
+ "step": 181
1278
+ },
1279
+ {
1280
+ "epoch": 5.056338028169014,
1281
+ "grad_norm": 0.24395267689073322,
1282
+ "learning_rate": 9.774874231860935e-07,
1283
+ "loss": 0.265809565782547,
1284
+ "step": 182
1285
+ },
1286
+ {
1287
+ "epoch": 5.084507042253521,
1288
+ "grad_norm": 0.2492192509436832,
1289
+ "learning_rate": 9.772414649388568e-07,
1290
+ "loss": 0.2792435884475708,
1291
+ "step": 183
1292
+ },
1293
+ {
1294
+ "epoch": 5.112676056338028,
1295
+ "grad_norm": 0.21622914380528604,
1296
+ "learning_rate": 9.769942052400235e-07,
1297
+ "loss": 0.26818716526031494,
1298
+ "step": 184
1299
+ },
1300
+ {
1301
+ "epoch": 5.140845070422535,
1302
+ "grad_norm": 0.27867715267424076,
1303
+ "learning_rate": 9.767456448427896e-07,
1304
+ "loss": 0.27086278796195984,
1305
+ "step": 185
1306
+ },
1307
+ {
1308
+ "epoch": 5.169014084507042,
1309
+ "grad_norm": 0.2346306080793482,
1310
+ "learning_rate": 9.764957845043135e-07,
1311
+ "loss": 0.2673496901988983,
1312
+ "step": 186
1313
+ },
1314
+ {
1315
+ "epoch": 5.197183098591549,
1316
+ "grad_norm": 0.23851487965670903,
1317
+ "learning_rate": 9.76244624985713e-07,
1318
+ "loss": 0.2633378803730011,
1319
+ "step": 187
1320
+ },
1321
+ {
1322
+ "epoch": 5.225352112676056,
1323
+ "grad_norm": 0.2362238221478819,
1324
+ "learning_rate": 9.759921670520634e-07,
1325
+ "loss": 0.27480238676071167,
1326
+ "step": 188
1327
+ },
1328
+ {
1329
+ "epoch": 5.253521126760563,
1330
+ "grad_norm": 0.23011578681816747,
1331
+ "learning_rate": 9.757384114723953e-07,
1332
+ "loss": 0.28158247470855713,
1333
+ "step": 189
1334
+ },
1335
+ {
1336
+ "epoch": 5.28169014084507,
1337
+ "grad_norm": 0.2323771506860616,
1338
+ "learning_rate": 9.754833590196926e-07,
1339
+ "loss": 0.2691372334957123,
1340
+ "step": 190
1341
+ },
1342
+ {
1343
+ "epoch": 5.309859154929577,
1344
+ "grad_norm": 0.23247640324241922,
1345
+ "learning_rate": 9.752270104708888e-07,
1346
+ "loss": 0.2666372060775757,
1347
+ "step": 191
1348
+ },
1349
+ {
1350
+ "epoch": 5.338028169014084,
1351
+ "grad_norm": 0.35005461804027077,
1352
+ "learning_rate": 9.749693666068663e-07,
1353
+ "loss": 0.2779178321361542,
1354
+ "step": 192
1355
+ },
1356
+ {
1357
+ "epoch": 5.366197183098592,
1358
+ "grad_norm": 0.29793203943964064,
1359
+ "learning_rate": 9.747104282124531e-07,
1360
+ "loss": 0.26953545212745667,
1361
+ "step": 193
1362
+ },
1363
+ {
1364
+ "epoch": 5.394366197183099,
1365
+ "grad_norm": 0.2262507700581054,
1366
+ "learning_rate": 9.744501960764203e-07,
1367
+ "loss": 0.26865124702453613,
1368
+ "step": 194
1369
+ },
1370
+ {
1371
+ "epoch": 5.422535211267606,
1372
+ "grad_norm": 0.21683849012894404,
1373
+ "learning_rate": 9.741886709914803e-07,
1374
+ "loss": 0.25939399003982544,
1375
+ "step": 195
1376
+ },
1377
+ {
1378
+ "epoch": 5.450704225352113,
1379
+ "grad_norm": 0.22595351538334174,
1380
+ "learning_rate": 9.739258537542835e-07,
1381
+ "loss": 0.2663502097129822,
1382
+ "step": 196
1383
+ },
1384
+ {
1385
+ "epoch": 5.47887323943662,
1386
+ "grad_norm": 0.22169050419160188,
1387
+ "learning_rate": 9.73661745165417e-07,
1388
+ "loss": 0.2786925435066223,
1389
+ "step": 197
1390
+ },
1391
+ {
1392
+ "epoch": 5.507042253521127,
1393
+ "grad_norm": 0.24492486958596588,
1394
+ "learning_rate": 9.733963460294015e-07,
1395
+ "loss": 0.27515316009521484,
1396
+ "step": 198
1397
+ },
1398
+ {
1399
+ "epoch": 5.535211267605634,
1400
+ "grad_norm": 0.22826949989900344,
1401
+ "learning_rate": 9.731296571546885e-07,
1402
+ "loss": 0.28115102648735046,
1403
+ "step": 199
1404
+ },
1405
+ {
1406
+ "epoch": 5.563380281690141,
1407
+ "grad_norm": 0.26522607771513945,
1408
+ "learning_rate": 9.728616793536587e-07,
1409
+ "loss": 0.2721824645996094,
1410
+ "step": 200
1411
+ },
1412
+ {
1413
+ "epoch": 5.591549295774648,
1414
+ "grad_norm": 0.22800471881703901,
1415
+ "learning_rate": 9.72592413442619e-07,
1416
+ "loss": 0.28054916858673096,
1417
+ "step": 201
1418
+ },
1419
+ {
1420
+ "epoch": 5.619718309859155,
1421
+ "grad_norm": 0.2347149530251392,
1422
+ "learning_rate": 9.723218602418e-07,
1423
+ "loss": 0.28160133957862854,
1424
+ "step": 202
1425
+ },
1426
+ {
1427
+ "epoch": 5.647887323943662,
1428
+ "grad_norm": 0.239273549722643,
1429
+ "learning_rate": 9.720500205753538e-07,
1430
+ "loss": 0.26327449083328247,
1431
+ "step": 203
1432
+ },
1433
+ {
1434
+ "epoch": 5.676056338028169,
1435
+ "grad_norm": 0.24387472862754722,
1436
+ "learning_rate": 9.717768952713511e-07,
1437
+ "loss": 0.2734866738319397,
1438
+ "step": 204
1439
+ },
1440
+ {
1441
+ "epoch": 5.704225352112676,
1442
+ "grad_norm": 0.23364964137411087,
1443
+ "learning_rate": 9.71502485161779e-07,
1444
+ "loss": 0.2678722143173218,
1445
+ "step": 205
1446
+ },
1447
+ {
1448
+ "epoch": 5.732394366197183,
1449
+ "grad_norm": 0.22168298331328018,
1450
+ "learning_rate": 9.71226791082538e-07,
1451
+ "loss": 0.28682976961135864,
1452
+ "step": 206
1453
+ },
1454
+ {
1455
+ "epoch": 5.76056338028169,
1456
+ "grad_norm": 0.22794997018956722,
1457
+ "learning_rate": 9.709498138734403e-07,
1458
+ "loss": 0.26408618688583374,
1459
+ "step": 207
1460
+ },
1461
+ {
1462
+ "epoch": 5.788732394366197,
1463
+ "grad_norm": 0.23082951390472506,
1464
+ "learning_rate": 9.706715543782064e-07,
1465
+ "loss": 0.26725077629089355,
1466
+ "step": 208
1467
+ },
1468
+ {
1469
+ "epoch": 5.816901408450704,
1470
+ "grad_norm": 0.23772477020760396,
1471
+ "learning_rate": 9.703920134444632e-07,
1472
+ "loss": 0.264972060918808,
1473
+ "step": 209
1474
+ },
1475
+ {
1476
+ "epoch": 5.845070422535211,
1477
+ "grad_norm": 0.24144546698053745,
1478
+ "learning_rate": 9.701111919237408e-07,
1479
+ "loss": 0.27345162630081177,
1480
+ "step": 210
1481
+ },
1482
+ {
1483
+ "epoch": 5.873239436619718,
1484
+ "grad_norm": 0.21694849932490126,
1485
+ "learning_rate": 9.698290906714702e-07,
1486
+ "loss": 0.25832873582839966,
1487
+ "step": 211
1488
+ },
1489
+ {
1490
+ "epoch": 5.901408450704225,
1491
+ "grad_norm": 0.23606790891410948,
1492
+ "learning_rate": 9.695457105469804e-07,
1493
+ "loss": 0.27317512035369873,
1494
+ "step": 212
1495
+ },
1496
+ {
1497
+ "epoch": 5.929577464788732,
1498
+ "grad_norm": 0.22036780479653179,
1499
+ "learning_rate": 9.69261052413497e-07,
1500
+ "loss": 0.278011292219162,
1501
+ "step": 213
1502
+ },
1503
+ {
1504
+ "epoch": 5.957746478873239,
1505
+ "grad_norm": 0.22667530947182388,
1506
+ "learning_rate": 9.689751171381377e-07,
1507
+ "loss": 0.2705678641796112,
1508
+ "step": 214
1509
+ },
1510
+ {
1511
+ "epoch": 5.985915492957746,
1512
+ "grad_norm": 0.21789301750027548,
1513
+ "learning_rate": 9.68687905591911e-07,
1514
+ "loss": 0.26159361004829407,
1515
+ "step": 215
1516
+ },
1517
+ {
1518
+ "epoch": 6.0,
1519
+ "grad_norm": 0.307035373509359,
1520
+ "learning_rate": 9.683994186497132e-07,
1521
+ "loss": 0.268825888633728,
1522
+ "step": 216
1523
+ },
1524
+ {
1525
+ "epoch": 6.028169014084507,
1526
+ "grad_norm": 0.3152032154597242,
1527
+ "learning_rate": 9.681096571903252e-07,
1528
+ "loss": 0.26456916332244873,
1529
+ "step": 217
1530
+ },
1531
+ {
1532
+ "epoch": 6.056338028169014,
1533
+ "grad_norm": 0.22257647731470015,
1534
+ "learning_rate": 9.67818622096411e-07,
1535
+ "loss": 0.25304824113845825,
1536
+ "step": 218
1537
+ },
1538
+ {
1539
+ "epoch": 6.084507042253521,
1540
+ "grad_norm": 0.22819800840776633,
1541
+ "learning_rate": 9.67526314254514e-07,
1542
+ "loss": 0.2657697796821594,
1543
+ "step": 219
1544
+ },
1545
+ {
1546
+ "epoch": 6.112676056338028,
1547
+ "grad_norm": 0.22326346608679778,
1548
+ "learning_rate": 9.672327345550543e-07,
1549
+ "loss": 0.2606521248817444,
1550
+ "step": 220
1551
+ },
1552
+ {
1553
+ "epoch": 6.140845070422535,
1554
+ "grad_norm": 0.24238210645266042,
1555
+ "learning_rate": 9.669378838923267e-07,
1556
+ "loss": 0.259613037109375,
1557
+ "step": 221
1558
+ },
1559
+ {
1560
+ "epoch": 6.169014084507042,
1561
+ "grad_norm": 0.23469955143149954,
1562
+ "learning_rate": 9.666417631644976e-07,
1563
+ "loss": 0.27692991495132446,
1564
+ "step": 222
1565
+ },
1566
+ {
1567
+ "epoch": 6.197183098591549,
1568
+ "grad_norm": 0.32987217731044594,
1569
+ "learning_rate": 9.66344373273602e-07,
1570
+ "loss": 0.2645450532436371,
1571
+ "step": 223
1572
+ },
1573
+ {
1574
+ "epoch": 6.225352112676056,
1575
+ "grad_norm": 0.22700888241362085,
1576
+ "learning_rate": 9.66045715125541e-07,
1577
+ "loss": 0.2646715044975281,
1578
+ "step": 224
1579
+ },
1580
+ {
1581
+ "epoch": 6.253521126760563,
1582
+ "grad_norm": 0.22957306298672545,
1583
+ "learning_rate": 9.657457896300791e-07,
1584
+ "loss": 0.24674871563911438,
1585
+ "step": 225
1586
+ },
1587
+ {
1588
+ "epoch": 6.28169014084507,
1589
+ "grad_norm": 0.23586354836136889,
1590
+ "learning_rate": 9.654445977008414e-07,
1591
+ "loss": 0.2587713599205017,
1592
+ "step": 226
1593
+ },
1594
+ {
1595
+ "epoch": 6.309859154929577,
1596
+ "grad_norm": 0.2285859030240268,
1597
+ "learning_rate": 9.651421402553108e-07,
1598
+ "loss": 0.2513459026813507,
1599
+ "step": 227
1600
+ },
1601
+ {
1602
+ "epoch": 6.338028169014084,
1603
+ "grad_norm": 0.24331804613486147,
1604
+ "learning_rate": 9.648384182148252e-07,
1605
+ "loss": 0.2715184688568115,
1606
+ "step": 228
1607
+ },
1608
+ {
1609
+ "epoch": 6.366197183098592,
1610
+ "grad_norm": 0.2305076454423525,
1611
+ "learning_rate": 9.645334325045745e-07,
1612
+ "loss": 0.2733456492424011,
1613
+ "step": 229
1614
+ },
1615
+ {
1616
+ "epoch": 6.394366197183099,
1617
+ "grad_norm": 0.24182818411808535,
1618
+ "learning_rate": 9.64227184053598e-07,
1619
+ "loss": 0.27919310331344604,
1620
+ "step": 230
1621
+ },
1622
+ {
1623
+ "epoch": 6.422535211267606,
1624
+ "grad_norm": 0.24680603573979737,
1625
+ "learning_rate": 9.63919673794782e-07,
1626
+ "loss": 0.26471200585365295,
1627
+ "step": 231
1628
+ },
1629
+ {
1630
+ "epoch": 6.450704225352113,
1631
+ "grad_norm": 0.24016603586966853,
1632
+ "learning_rate": 9.636109026648554e-07,
1633
+ "loss": 0.26608601212501526,
1634
+ "step": 232
1635
+ },
1636
+ {
1637
+ "epoch": 6.47887323943662,
1638
+ "grad_norm": 0.23221033716137462,
1639
+ "learning_rate": 9.633008716043892e-07,
1640
+ "loss": 0.2712378203868866,
1641
+ "step": 233
1642
+ },
1643
+ {
1644
+ "epoch": 6.507042253521127,
1645
+ "grad_norm": 0.22946156470598095,
1646
+ "learning_rate": 9.629895815577915e-07,
1647
+ "loss": 0.2693229913711548,
1648
+ "step": 234
1649
+ },
1650
+ {
1651
+ "epoch": 6.535211267605634,
1652
+ "grad_norm": 0.24256759776346887,
1653
+ "learning_rate": 9.626770334733058e-07,
1654
+ "loss": 0.25974130630493164,
1655
+ "step": 235
1656
+ },
1657
+ {
1658
+ "epoch": 6.563380281690141,
1659
+ "grad_norm": 0.2284537050943504,
1660
+ "learning_rate": 9.623632283030077e-07,
1661
+ "loss": 0.27251625061035156,
1662
+ "step": 236
1663
+ },
1664
+ {
1665
+ "epoch": 6.591549295774648,
1666
+ "grad_norm": 0.22616819126049306,
1667
+ "learning_rate": 9.620481670028026e-07,
1668
+ "loss": 0.26490986347198486,
1669
+ "step": 237
1670
+ },
1671
+ {
1672
+ "epoch": 6.619718309859155,
1673
+ "grad_norm": 0.22191171035923044,
1674
+ "learning_rate": 9.617318505324212e-07,
1675
+ "loss": 0.25655436515808105,
1676
+ "step": 238
1677
+ },
1678
+ {
1679
+ "epoch": 6.647887323943662,
1680
+ "grad_norm": 0.23707447655959804,
1681
+ "learning_rate": 9.614142798554186e-07,
1682
+ "loss": 0.2742191553115845,
1683
+ "step": 239
1684
+ },
1685
+ {
1686
+ "epoch": 6.676056338028169,
1687
+ "grad_norm": 0.2553264029309018,
1688
+ "learning_rate": 9.610954559391704e-07,
1689
+ "loss": 0.25515586137771606,
1690
+ "step": 240
1691
+ },
1692
+ {
1693
+ "epoch": 6.704225352112676,
1694
+ "grad_norm": 0.24273871332366953,
1695
+ "learning_rate": 9.607753797548691e-07,
1696
+ "loss": 0.2646741271018982,
1697
+ "step": 241
1698
+ },
1699
+ {
1700
+ "epoch": 6.732394366197183,
1701
+ "grad_norm": 0.2333343519162289,
1702
+ "learning_rate": 9.604540522775227e-07,
1703
+ "loss": 0.26033157110214233,
1704
+ "step": 242
1705
+ },
1706
+ {
1707
+ "epoch": 6.76056338028169,
1708
+ "grad_norm": 0.22965997751864728,
1709
+ "learning_rate": 9.601314744859504e-07,
1710
+ "loss": 0.27026018500328064,
1711
+ "step": 243
1712
+ },
1713
+ {
1714
+ "epoch": 6.788732394366197,
1715
+ "grad_norm": 0.23498879472224687,
1716
+ "learning_rate": 9.598076473627796e-07,
1717
+ "loss": 0.26615339517593384,
1718
+ "step": 244
1719
+ },
1720
+ {
1721
+ "epoch": 6.816901408450704,
1722
+ "grad_norm": 0.22789781362841657,
1723
+ "learning_rate": 9.594825718944444e-07,
1724
+ "loss": 0.25952741503715515,
1725
+ "step": 245
1726
+ },
1727
+ {
1728
+ "epoch": 6.845070422535211,
1729
+ "grad_norm": 0.4285246410791299,
1730
+ "learning_rate": 9.59156249071181e-07,
1731
+ "loss": 0.2581027150154114,
1732
+ "step": 246
1733
+ },
1734
+ {
1735
+ "epoch": 6.873239436619718,
1736
+ "grad_norm": 0.22610447167940628,
1737
+ "learning_rate": 9.588286798870248e-07,
1738
+ "loss": 0.27599114179611206,
1739
+ "step": 247
1740
+ },
1741
+ {
1742
+ "epoch": 6.901408450704225,
1743
+ "grad_norm": 0.22882782061457732,
1744
+ "learning_rate": 9.58499865339809e-07,
1745
+ "loss": 0.2618204653263092,
1746
+ "step": 248
1747
+ },
1748
+ {
1749
+ "epoch": 6.929577464788732,
1750
+ "grad_norm": 0.26429160943599095,
1751
+ "learning_rate": 9.581698064311592e-07,
1752
+ "loss": 0.2596527338027954,
1753
+ "step": 249
1754
+ },
1755
+ {
1756
+ "epoch": 6.957746478873239,
1757
+ "grad_norm": 0.30324449730772324,
1758
+ "learning_rate": 9.578385041664925e-07,
1759
+ "loss": 0.26591378450393677,
1760
+ "step": 250
1761
+ },
1762
+ {
1763
+ "epoch": 6.985915492957746,
1764
+ "grad_norm": 0.24030702361568637,
1765
+ "learning_rate": 9.575059595550127e-07,
1766
+ "loss": 0.2602125406265259,
1767
+ "step": 251
1768
+ },
1769
+ {
1770
+ "epoch": 7.0,
1771
+ "grad_norm": 0.3109240011602677,
1772
+ "learning_rate": 9.571721736097088e-07,
1773
+ "loss": 0.2538529634475708,
1774
+ "step": 252
1775
+ },
1776
+ {
1777
+ "epoch": 7.028169014084507,
1778
+ "grad_norm": 0.2296122757795814,
1779
+ "learning_rate": 9.568371473473503e-07,
1780
+ "loss": 0.27390003204345703,
1781
+ "step": 253
1782
+ },
1783
+ {
1784
+ "epoch": 7.056338028169014,
1785
+ "grad_norm": 0.23108143057622355,
1786
+ "learning_rate": 9.565008817884854e-07,
1787
+ "loss": 0.262205570936203,
1788
+ "step": 254
1789
+ },
1790
+ {
1791
+ "epoch": 7.084507042253521,
1792
+ "grad_norm": 0.3057597754632391,
1793
+ "learning_rate": 9.561633779574372e-07,
1794
+ "loss": 0.2676515579223633,
1795
+ "step": 255
1796
+ },
1797
+ {
1798
+ "epoch": 7.112676056338028,
1799
+ "grad_norm": 0.2384413247157669,
1800
+ "learning_rate": 9.55824636882301e-07,
1801
+ "loss": 0.25623461604118347,
1802
+ "step": 256
1803
+ },
1804
+ {
1805
+ "epoch": 7.140845070422535,
1806
+ "grad_norm": 0.22577575110661233,
1807
+ "learning_rate": 9.554846595949413e-07,
1808
+ "loss": 0.25066760182380676,
1809
+ "step": 257
1810
+ },
1811
+ {
1812
+ "epoch": 7.169014084507042,
1813
+ "grad_norm": 0.22810299013323876,
1814
+ "learning_rate": 9.55143447130987e-07,
1815
+ "loss": 0.25772857666015625,
1816
+ "step": 258
1817
+ },
1818
+ {
1819
+ "epoch": 7.197183098591549,
1820
+ "grad_norm": 0.2866432998882861,
1821
+ "learning_rate": 9.54801000529831e-07,
1822
+ "loss": 0.25286948680877686,
1823
+ "step": 259
1824
+ },
1825
+ {
1826
+ "epoch": 7.225352112676056,
1827
+ "grad_norm": 0.27677955881524746,
1828
+ "learning_rate": 9.54457320834625e-07,
1829
+ "loss": 0.2480214238166809,
1830
+ "step": 260
1831
+ },
1832
+ {
1833
+ "epoch": 7.253521126760563,
1834
+ "grad_norm": 0.23561125478874487,
1835
+ "learning_rate": 9.54112409092277e-07,
1836
+ "loss": 0.273529589176178,
1837
+ "step": 261
1838
+ },
1839
+ {
1840
+ "epoch": 7.28169014084507,
1841
+ "grad_norm": 0.23173643804892863,
1842
+ "learning_rate": 9.537662663534477e-07,
1843
+ "loss": 0.2518290877342224,
1844
+ "step": 262
1845
+ },
1846
+ {
1847
+ "epoch": 7.309859154929577,
1848
+ "grad_norm": 0.2256175554720283,
1849
+ "learning_rate": 9.534188936725483e-07,
1850
+ "loss": 0.2571009695529938,
1851
+ "step": 263
1852
+ },
1853
+ {
1854
+ "epoch": 7.338028169014084,
1855
+ "grad_norm": 0.2355621515439768,
1856
+ "learning_rate": 9.530702921077358e-07,
1857
+ "loss": 0.25834065675735474,
1858
+ "step": 264
1859
+ },
1860
+ {
1861
+ "epoch": 7.366197183098592,
1862
+ "grad_norm": 0.2228326754230289,
1863
+ "learning_rate": 9.527204627209112e-07,
1864
+ "loss": 0.25286680459976196,
1865
+ "step": 265
1866
+ },
1867
+ {
1868
+ "epoch": 7.394366197183099,
1869
+ "grad_norm": 0.23194189643389376,
1870
+ "learning_rate": 9.523694065777156e-07,
1871
+ "loss": 0.2616320848464966,
1872
+ "step": 266
1873
+ },
1874
+ {
1875
+ "epoch": 7.422535211267606,
1876
+ "grad_norm": 0.23212710466470063,
1877
+ "learning_rate": 9.520171247475268e-07,
1878
+ "loss": 0.2550612688064575,
1879
+ "step": 267
1880
+ },
1881
+ {
1882
+ "epoch": 7.450704225352113,
1883
+ "grad_norm": 0.2630254038278628,
1884
+ "learning_rate": 9.516636183034564e-07,
1885
+ "loss": 0.2545396089553833,
1886
+ "step": 268
1887
+ },
1888
+ {
1889
+ "epoch": 7.47887323943662,
1890
+ "grad_norm": 0.22233881792271545,
1891
+ "learning_rate": 9.513088883223463e-07,
1892
+ "loss": 0.244547039270401,
1893
+ "step": 269
1894
+ },
1895
+ {
1896
+ "epoch": 7.507042253521127,
1897
+ "grad_norm": 0.24058852766292496,
1898
+ "learning_rate": 9.509529358847654e-07,
1899
+ "loss": 0.258344829082489,
1900
+ "step": 270
1901
+ },
1902
+ {
1903
+ "epoch": 7.535211267605634,
1904
+ "grad_norm": 0.2591231329098907,
1905
+ "learning_rate": 9.505957620750069e-07,
1906
+ "loss": 0.2509728670120239,
1907
+ "step": 271
1908
+ },
1909
+ {
1910
+ "epoch": 7.563380281690141,
1911
+ "grad_norm": 0.22516830746766386,
1912
+ "learning_rate": 9.502373679810839e-07,
1913
+ "loss": 0.260656476020813,
1914
+ "step": 272
1915
+ },
1916
+ {
1917
+ "epoch": 7.591549295774648,
1918
+ "grad_norm": 0.32992687123033926,
1919
+ "learning_rate": 9.49877754694727e-07,
1920
+ "loss": 0.2461855411529541,
1921
+ "step": 273
1922
+ },
1923
+ {
1924
+ "epoch": 7.619718309859155,
1925
+ "grad_norm": 0.22517930953114113,
1926
+ "learning_rate": 9.495169233113806e-07,
1927
+ "loss": 0.26402074098587036,
1928
+ "step": 274
1929
+ },
1930
+ {
1931
+ "epoch": 7.647887323943662,
1932
+ "grad_norm": 0.2303015677956865,
1933
+ "learning_rate": 9.491548749301997e-07,
1934
+ "loss": 0.2625475823879242,
1935
+ "step": 275
1936
+ },
1937
+ {
1938
+ "epoch": 7.676056338028169,
1939
+ "grad_norm": 0.24736115424944397,
1940
+ "learning_rate": 9.487916106540465e-07,
1941
+ "loss": 0.261366605758667,
1942
+ "step": 276
1943
+ },
1944
+ {
1945
+ "epoch": 7.704225352112676,
1946
+ "grad_norm": 0.3054635100649164,
1947
+ "learning_rate": 9.484271315894871e-07,
1948
+ "loss": 0.2583233714103699,
1949
+ "step": 277
1950
+ },
1951
+ {
1952
+ "epoch": 7.732394366197183,
1953
+ "grad_norm": 0.23888784253517656,
1954
+ "learning_rate": 9.480614388467877e-07,
1955
+ "loss": 0.26100364327430725,
1956
+ "step": 278
1957
+ },
1958
+ {
1959
+ "epoch": 7.76056338028169,
1960
+ "grad_norm": 0.23035518389984047,
1961
+ "learning_rate": 9.47694533539912e-07,
1962
+ "loss": 0.25427085161209106,
1963
+ "step": 279
1964
+ },
1965
+ {
1966
+ "epoch": 7.788732394366197,
1967
+ "grad_norm": 0.2397167748787147,
1968
+ "learning_rate": 9.473264167865171e-07,
1969
+ "loss": 0.26791495084762573,
1970
+ "step": 280
1971
+ },
1972
+ {
1973
+ "epoch": 7.816901408450704,
1974
+ "grad_norm": 0.2324670883748479,
1975
+ "learning_rate": 9.469570897079504e-07,
1976
+ "loss": 0.25854068994522095,
1977
+ "step": 281
1978
+ },
1979
+ {
1980
+ "epoch": 7.845070422535211,
1981
+ "grad_norm": 0.2407835066696354,
1982
+ "learning_rate": 9.465865534292464e-07,
1983
+ "loss": 0.25427621603012085,
1984
+ "step": 282
1985
+ },
1986
+ {
1987
+ "epoch": 7.873239436619718,
1988
+ "grad_norm": 0.24566208023836886,
1989
+ "learning_rate": 9.462148090791228e-07,
1990
+ "loss": 0.25798651576042175,
1991
+ "step": 283
1992
+ },
1993
+ {
1994
+ "epoch": 7.901408450704225,
1995
+ "grad_norm": 0.25230839039720004,
1996
+ "learning_rate": 9.458418577899774e-07,
1997
+ "loss": 0.2794758379459381,
1998
+ "step": 284
1999
+ },
2000
+ {
2001
+ "epoch": 7.929577464788732,
2002
+ "grad_norm": 0.22946111074733544,
2003
+ "learning_rate": 9.454677006978842e-07,
2004
+ "loss": 0.2522087097167969,
2005
+ "step": 285
2006
+ },
2007
+ {
2008
+ "epoch": 7.957746478873239,
2009
+ "grad_norm": 0.23483999818710283,
2010
+ "learning_rate": 9.450923389425911e-07,
2011
+ "loss": 0.24993300437927246,
2012
+ "step": 286
2013
+ },
2014
+ {
2015
+ "epoch": 7.985915492957746,
2016
+ "grad_norm": 0.23222784962960594,
2017
+ "learning_rate": 9.44715773667515e-07,
2018
+ "loss": 0.2578023672103882,
2019
+ "step": 287
2020
+ },
2021
+ {
2022
+ "epoch": 8.0,
2023
+ "grad_norm": 0.33496486871987224,
2024
+ "learning_rate": 9.443380060197385e-07,
2025
+ "loss": 0.2590055763721466,
2026
+ "step": 288
2027
+ },
2028
+ {
2029
+ "epoch": 8.028169014084508,
2030
+ "grad_norm": 0.22978114673982541,
2031
+ "learning_rate": 9.43959037150008e-07,
2032
+ "loss": 0.2533671259880066,
2033
+ "step": 289
2034
+ },
2035
+ {
2036
+ "epoch": 8.056338028169014,
2037
+ "grad_norm": 0.24537281724911067,
2038
+ "learning_rate": 9.43578868212728e-07,
2039
+ "loss": 0.2540019154548645,
2040
+ "step": 290
2041
+ },
2042
+ {
2043
+ "epoch": 8.084507042253522,
2044
+ "grad_norm": 0.23952878163512892,
2045
+ "learning_rate": 9.431975003659594e-07,
2046
+ "loss": 0.2519161105155945,
2047
+ "step": 291
2048
+ },
2049
+ {
2050
+ "epoch": 8.112676056338028,
2051
+ "grad_norm": 0.22410269302607416,
2052
+ "learning_rate": 9.428149347714143e-07,
2053
+ "loss": 0.2534847855567932,
2054
+ "step": 292
2055
+ },
2056
+ {
2057
+ "epoch": 8.140845070422536,
2058
+ "grad_norm": 0.22515357416422843,
2059
+ "learning_rate": 9.424311725944543e-07,
2060
+ "loss": 0.2480982393026352,
2061
+ "step": 293
2062
+ },
2063
+ {
2064
+ "epoch": 8.169014084507042,
2065
+ "grad_norm": 0.22463212198832308,
2066
+ "learning_rate": 9.420462150040852e-07,
2067
+ "loss": 0.25113674998283386,
2068
+ "step": 294
2069
+ },
2070
+ {
2071
+ "epoch": 8.19718309859155,
2072
+ "grad_norm": 0.24243654364581727,
2073
+ "learning_rate": 9.416600631729548e-07,
2074
+ "loss": 0.2588122487068176,
2075
+ "step": 295
2076
+ },
2077
+ {
2078
+ "epoch": 8.225352112676056,
2079
+ "grad_norm": 0.23662758753633675,
2080
+ "learning_rate": 9.412727182773486e-07,
2081
+ "loss": 0.2564491033554077,
2082
+ "step": 296
2083
+ },
2084
+ {
2085
+ "epoch": 8.253521126760564,
2086
+ "grad_norm": 0.22745130948641154,
2087
+ "learning_rate": 9.408841814971861e-07,
2088
+ "loss": 0.25671282410621643,
2089
+ "step": 297
2090
+ },
2091
+ {
2092
+ "epoch": 8.28169014084507,
2093
+ "grad_norm": 0.23421703627414328,
2094
+ "learning_rate": 9.404944540160177e-07,
2095
+ "loss": 0.25537922978401184,
2096
+ "step": 298
2097
+ },
2098
+ {
2099
+ "epoch": 8.309859154929578,
2100
+ "grad_norm": 0.25808749865907027,
2101
+ "learning_rate": 9.401035370210212e-07,
2102
+ "loss": 0.2609266936779022,
2103
+ "step": 299
2104
+ },
2105
+ {
2106
+ "epoch": 8.338028169014084,
2107
+ "grad_norm": 0.26628076978008886,
2108
+ "learning_rate": 9.397114317029974e-07,
2109
+ "loss": 0.26676198840141296,
2110
+ "step": 300
2111
+ },
2112
+ {
2113
+ "epoch": 8.366197183098592,
2114
+ "grad_norm": 0.24806292816726846,
2115
+ "learning_rate": 9.393181392563669e-07,
2116
+ "loss": 0.2629510462284088,
2117
+ "step": 301
2118
+ },
2119
+ {
2120
+ "epoch": 8.394366197183098,
2121
+ "grad_norm": 0.2450141484218029,
2122
+ "learning_rate": 9.38923660879167e-07,
2123
+ "loss": 0.24178162217140198,
2124
+ "step": 302
2125
+ },
2126
+ {
2127
+ "epoch": 8.422535211267606,
2128
+ "grad_norm": 0.22586508580682582,
2129
+ "learning_rate": 9.385279977730472e-07,
2130
+ "loss": 0.2511584460735321,
2131
+ "step": 303
2132
+ },
2133
+ {
2134
+ "epoch": 8.450704225352112,
2135
+ "grad_norm": 0.2351453280752253,
2136
+ "learning_rate": 9.381311511432658e-07,
2137
+ "loss": 0.2653919458389282,
2138
+ "step": 304
2139
+ },
2140
+ {
2141
+ "epoch": 8.47887323943662,
2142
+ "grad_norm": 0.241646635077158,
2143
+ "learning_rate": 9.377331221986866e-07,
2144
+ "loss": 0.24603968858718872,
2145
+ "step": 305
2146
+ },
2147
+ {
2148
+ "epoch": 8.507042253521126,
2149
+ "grad_norm": 0.2340015157165136,
2150
+ "learning_rate": 9.373339121517746e-07,
2151
+ "loss": 0.2578321695327759,
2152
+ "step": 306
2153
+ },
2154
+ {
2155
+ "epoch": 8.535211267605634,
2156
+ "grad_norm": 0.2545851654819562,
2157
+ "learning_rate": 9.36933522218593e-07,
2158
+ "loss": 0.2556232213973999,
2159
+ "step": 307
2160
+ },
2161
+ {
2162
+ "epoch": 8.56338028169014,
2163
+ "grad_norm": 0.2402873441824482,
2164
+ "learning_rate": 9.36531953618799e-07,
2165
+ "loss": 0.24149319529533386,
2166
+ "step": 308
2167
+ },
2168
+ {
2169
+ "epoch": 8.591549295774648,
2170
+ "grad_norm": 0.22732048023459145,
2171
+ "learning_rate": 9.361292075756401e-07,
2172
+ "loss": 0.2583070695400238,
2173
+ "step": 309
2174
+ },
2175
+ {
2176
+ "epoch": 8.619718309859154,
2177
+ "grad_norm": 0.23686044423247699,
2178
+ "learning_rate": 9.357252853159505e-07,
2179
+ "loss": 0.2552328407764435,
2180
+ "step": 310
2181
+ },
2182
+ {
2183
+ "epoch": 8.647887323943662,
2184
+ "grad_norm": 0.248000780112058,
2185
+ "learning_rate": 9.353201880701477e-07,
2186
+ "loss": 0.2555694878101349,
2187
+ "step": 311
2188
+ },
2189
+ {
2190
+ "epoch": 8.676056338028168,
2191
+ "grad_norm": 0.23819636245003398,
2192
+ "learning_rate": 9.34913917072228e-07,
2193
+ "loss": 0.2535495162010193,
2194
+ "step": 312
2195
+ },
2196
+ {
2197
+ "epoch": 8.704225352112676,
2198
+ "grad_norm": 0.23289151649857873,
2199
+ "learning_rate": 9.345064735597633e-07,
2200
+ "loss": 0.23895904421806335,
2201
+ "step": 313
2202
+ },
2203
+ {
2204
+ "epoch": 8.732394366197184,
2205
+ "grad_norm": 0.24456775846548265,
2206
+ "learning_rate": 9.340978587738972e-07,
2207
+ "loss": 0.24082013964653015,
2208
+ "step": 314
2209
+ },
2210
+ {
2211
+ "epoch": 8.76056338028169,
2212
+ "grad_norm": 0.24111494311817017,
2213
+ "learning_rate": 9.336880739593415e-07,
2214
+ "loss": 0.2520008683204651,
2215
+ "step": 315
2216
+ },
2217
+ {
2218
+ "epoch": 8.788732394366198,
2219
+ "grad_norm": 0.23455464565416306,
2220
+ "learning_rate": 9.332771203643714e-07,
2221
+ "loss": 0.24615193903446198,
2222
+ "step": 316
2223
+ },
2224
+ {
2225
+ "epoch": 8.816901408450704,
2226
+ "grad_norm": 0.23820264470002048,
2227
+ "learning_rate": 9.328649992408231e-07,
2228
+ "loss": 0.24194085597991943,
2229
+ "step": 317
2230
+ },
2231
+ {
2232
+ "epoch": 8.845070422535212,
2233
+ "grad_norm": 0.2509852236357078,
2234
+ "learning_rate": 9.324517118440888e-07,
2235
+ "loss": 0.2486490160226822,
2236
+ "step": 318
2237
+ },
2238
+ {
2239
+ "epoch": 8.873239436619718,
2240
+ "grad_norm": 0.2364300073153734,
2241
+ "learning_rate": 9.320372594331137e-07,
2242
+ "loss": 0.26214224100112915,
2243
+ "step": 319
2244
+ },
2245
+ {
2246
+ "epoch": 8.901408450704226,
2247
+ "grad_norm": 0.23707535806054583,
2248
+ "learning_rate": 9.316216432703916e-07,
2249
+ "loss": 0.2599393129348755,
2250
+ "step": 320
2251
+ },
2252
+ {
2253
+ "epoch": 8.929577464788732,
2254
+ "grad_norm": 0.2623964552394443,
2255
+ "learning_rate": 9.312048646219617e-07,
2256
+ "loss": 0.2490779161453247,
2257
+ "step": 321
2258
+ },
2259
+ {
2260
+ "epoch": 8.95774647887324,
2261
+ "grad_norm": 0.3409018820717708,
2262
+ "learning_rate": 9.307869247574038e-07,
2263
+ "loss": 0.24981269240379333,
2264
+ "step": 322
2265
+ },
2266
+ {
2267
+ "epoch": 8.985915492957746,
2268
+ "grad_norm": 0.23275952122730492,
2269
+ "learning_rate": 9.303678249498352e-07,
2270
+ "loss": 0.24256354570388794,
2271
+ "step": 323
2272
+ },
2273
+ {
2274
+ "epoch": 9.0,
2275
+ "grad_norm": 0.3314779892334818,
2276
+ "learning_rate": 9.299475664759068e-07,
2277
+ "loss": 0.2582060396671295,
2278
+ "step": 324
2279
+ },
2280
+ {
2281
+ "epoch": 9.028169014084508,
2282
+ "grad_norm": 0.22855183478172078,
2283
+ "learning_rate": 9.295261506157985e-07,
2284
+ "loss": 0.24924015998840332,
2285
+ "step": 325
2286
+ },
2287
+ {
2288
+ "epoch": 9.056338028169014,
2289
+ "grad_norm": 0.23011630291450727,
2290
+ "learning_rate": 9.291035786532163e-07,
2291
+ "loss": 0.24119523167610168,
2292
+ "step": 326
2293
+ },
2294
+ {
2295
+ "epoch": 9.084507042253522,
2296
+ "grad_norm": 0.25455626881208104,
2297
+ "learning_rate": 9.286798518753878e-07,
2298
+ "loss": 0.2545008659362793,
2299
+ "step": 327
2300
+ },
2301
+ {
2302
+ "epoch": 9.112676056338028,
2303
+ "grad_norm": 0.2470912179191389,
2304
+ "learning_rate": 9.282549715730579e-07,
2305
+ "loss": 0.25637179613113403,
2306
+ "step": 328
2307
+ },
2308
+ {
2309
+ "epoch": 9.140845070422536,
2310
+ "grad_norm": 0.2361076856223727,
2311
+ "learning_rate": 9.278289390404859e-07,
2312
+ "loss": 0.24920859932899475,
2313
+ "step": 329
2314
+ },
2315
+ {
2316
+ "epoch": 9.169014084507042,
2317
+ "grad_norm": 0.24962180209080553,
2318
+ "learning_rate": 9.274017555754407e-07,
2319
+ "loss": 0.25139665603637695,
2320
+ "step": 330
2321
+ },
2322
+ {
2323
+ "epoch": 9.19718309859155,
2324
+ "grad_norm": 0.2661403261625342,
2325
+ "learning_rate": 9.269734224791974e-07,
2326
+ "loss": 0.25162649154663086,
2327
+ "step": 331
2328
+ },
2329
+ {
2330
+ "epoch": 9.225352112676056,
2331
+ "grad_norm": 0.23321448395515867,
2332
+ "learning_rate": 9.265439410565328e-07,
2333
+ "loss": 0.2531318962574005,
2334
+ "step": 332
2335
+ },
2336
+ {
2337
+ "epoch": 9.253521126760564,
2338
+ "grad_norm": 0.2489447660723762,
2339
+ "learning_rate": 9.261133126157217e-07,
2340
+ "loss": 0.2439672201871872,
2341
+ "step": 333
2342
+ },
2343
+ {
2344
+ "epoch": 9.28169014084507,
2345
+ "grad_norm": 0.24654854069751458,
2346
+ "learning_rate": 9.256815384685328e-07,
2347
+ "loss": 0.24593070149421692,
2348
+ "step": 334
2349
+ },
2350
+ {
2351
+ "epoch": 9.309859154929578,
2352
+ "grad_norm": 0.24922607082895212,
2353
+ "learning_rate": 9.252486199302256e-07,
2354
+ "loss": 0.24202078580856323,
2355
+ "step": 335
2356
+ },
2357
+ {
2358
+ "epoch": 9.338028169014084,
2359
+ "grad_norm": 0.23810892907222608,
2360
+ "learning_rate": 9.248145583195447e-07,
2361
+ "loss": 0.24281273782253265,
2362
+ "step": 336
2363
+ },
2364
+ {
2365
+ "epoch": 9.366197183098592,
2366
+ "grad_norm": 0.2414836664150074,
2367
+ "learning_rate": 9.243793549587171e-07,
2368
+ "loss": 0.24529415369033813,
2369
+ "step": 337
2370
+ },
2371
+ {
2372
+ "epoch": 9.394366197183098,
2373
+ "grad_norm": 0.45146527719151147,
2374
+ "learning_rate": 9.239430111734476e-07,
2375
+ "loss": 0.24313722550868988,
2376
+ "step": 338
2377
+ },
2378
+ {
2379
+ "epoch": 9.422535211267606,
2380
+ "grad_norm": 0.2513687754640395,
2381
+ "learning_rate": 9.235055282929153e-07,
2382
+ "loss": 0.24014408886432648,
2383
+ "step": 339
2384
+ },
2385
+ {
2386
+ "epoch": 9.450704225352112,
2387
+ "grad_norm": 0.2464123403137622,
2388
+ "learning_rate": 9.230669076497687e-07,
2389
+ "loss": 0.25664836168289185,
2390
+ "step": 340
2391
+ },
2392
+ {
2393
+ "epoch": 9.47887323943662,
2394
+ "grad_norm": 0.25003507348604787,
2395
+ "learning_rate": 9.226271505801224e-07,
2396
+ "loss": 0.25439685583114624,
2397
+ "step": 341
2398
+ },
2399
+ {
2400
+ "epoch": 9.507042253521126,
2401
+ "grad_norm": 0.2460113899454725,
2402
+ "learning_rate": 9.221862584235526e-07,
2403
+ "loss": 0.2463434338569641,
2404
+ "step": 342
2405
+ },
2406
+ {
2407
+ "epoch": 9.535211267605634,
2408
+ "grad_norm": 0.29434373718545254,
2409
+ "learning_rate": 9.217442325230936e-07,
2410
+ "loss": 0.2347492277622223,
2411
+ "step": 343
2412
+ },
2413
+ {
2414
+ "epoch": 9.56338028169014,
2415
+ "grad_norm": 0.2426325292229501,
2416
+ "learning_rate": 9.213010742252327e-07,
2417
+ "loss": 0.2540351450443268,
2418
+ "step": 344
2419
+ },
2420
+ {
2421
+ "epoch": 9.591549295774648,
2422
+ "grad_norm": 0.23873379819321794,
2423
+ "learning_rate": 9.208567848799069e-07,
2424
+ "loss": 0.23636561632156372,
2425
+ "step": 345
2426
+ },
2427
+ {
2428
+ "epoch": 9.619718309859154,
2429
+ "grad_norm": 0.2536452901368959,
2430
+ "learning_rate": 9.204113658404989e-07,
2431
+ "loss": 0.2521906793117523,
2432
+ "step": 346
2433
+ },
2434
+ {
2435
+ "epoch": 9.647887323943662,
2436
+ "grad_norm": 0.23819082363894303,
2437
+ "learning_rate": 9.199648184638318e-07,
2438
+ "loss": 0.23743756115436554,
2439
+ "step": 347
2440
+ },
2441
+ {
2442
+ "epoch": 9.676056338028168,
2443
+ "grad_norm": 0.24809194025801815,
2444
+ "learning_rate": 9.195171441101668e-07,
2445
+ "loss": 0.2550021708011627,
2446
+ "step": 348
2447
+ },
2448
+ {
2449
+ "epoch": 9.704225352112676,
2450
+ "grad_norm": 0.2407861067961907,
2451
+ "learning_rate": 9.190683441431974e-07,
2452
+ "loss": 0.23909378051757812,
2453
+ "step": 349
2454
+ },
2455
+ {
2456
+ "epoch": 9.732394366197184,
2457
+ "grad_norm": 0.25194809703933585,
2458
+ "learning_rate": 9.186184199300463e-07,
2459
+ "loss": 0.2386261522769928,
2460
+ "step": 350
2461
+ },
2462
+ {
2463
+ "epoch": 9.76056338028169,
2464
+ "grad_norm": 0.3109079135054994,
2465
+ "learning_rate": 9.181673728412605e-07,
2466
+ "loss": 0.24811868369579315,
2467
+ "step": 351
2468
+ },
2469
+ {
2470
+ "epoch": 9.788732394366198,
2471
+ "grad_norm": 0.24849445193131672,
2472
+ "learning_rate": 9.177152042508077e-07,
2473
+ "loss": 0.24510633945465088,
2474
+ "step": 352
2475
+ },
2476
+ {
2477
+ "epoch": 9.816901408450704,
2478
+ "grad_norm": 0.25056831997285267,
2479
+ "learning_rate": 9.17261915536072e-07,
2480
+ "loss": 0.2476453185081482,
2481
+ "step": 353
2482
+ },
2483
+ {
2484
+ "epoch": 9.845070422535212,
2485
+ "grad_norm": 0.24753809085881498,
2486
+ "learning_rate": 9.168075080778494e-07,
2487
+ "loss": 0.24052247405052185,
2488
+ "step": 354
2489
+ },
2490
+ {
2491
+ "epoch": 9.873239436619718,
2492
+ "grad_norm": 0.27322821616611875,
2493
+ "learning_rate": 9.163519832603436e-07,
2494
+ "loss": 0.2500559687614441,
2495
+ "step": 355
2496
+ },
2497
+ {
2498
+ "epoch": 9.901408450704226,
2499
+ "grad_norm": 0.24879142424699366,
2500
+ "learning_rate": 9.158953424711624e-07,
2501
+ "loss": 0.26202425360679626,
2502
+ "step": 356
2503
+ },
2504
+ {
2505
+ "epoch": 9.929577464788732,
2506
+ "grad_norm": 0.2566208852525709,
2507
+ "learning_rate": 9.154375871013128e-07,
2508
+ "loss": 0.24967315793037415,
2509
+ "step": 357
2510
+ },
2511
+ {
2512
+ "epoch": 9.95774647887324,
2513
+ "grad_norm": 0.2544024404845266,
2514
+ "learning_rate": 9.149787185451969e-07,
2515
+ "loss": 0.25556012988090515,
2516
+ "step": 358
2517
+ },
2518
+ {
2519
+ "epoch": 9.985915492957746,
2520
+ "grad_norm": 0.2694091003451415,
2521
+ "learning_rate": 9.145187382006081e-07,
2522
+ "loss": 0.24726629257202148,
2523
+ "step": 359
2524
+ },
2525
+ {
2526
+ "epoch": 10.0,
2527
+ "grad_norm": 0.3663693099814081,
2528
+ "learning_rate": 9.140576474687263e-07,
2529
+ "loss": 0.26241111755371094,
2530
+ "step": 360
2531
+ },
2532
+ {
2533
+ "epoch": 10.028169014084508,
2534
+ "grad_norm": 0.23090643712953313,
2535
+ "learning_rate": 9.135954477541137e-07,
2536
+ "loss": 0.24851879477500916,
2537
+ "step": 361
2538
+ },
2539
+ {
2540
+ "epoch": 10.056338028169014,
2541
+ "grad_norm": 0.2616114206455057,
2542
+ "learning_rate": 9.131321404647109e-07,
2543
+ "loss": 0.25464296340942383,
2544
+ "step": 362
2545
+ },
2546
+ {
2547
+ "epoch": 10.084507042253522,
2548
+ "grad_norm": 1.0513495732621954,
2549
+ "learning_rate": 9.126677270118322e-07,
2550
+ "loss": 0.2508964538574219,
2551
+ "step": 363
2552
+ },
2553
+ {
2554
+ "epoch": 10.112676056338028,
2555
+ "grad_norm": 0.2430762392307408,
2556
+ "learning_rate": 9.122022088101613e-07,
2557
+ "loss": 0.23258915543556213,
2558
+ "step": 364
2559
+ },
2560
+ {
2561
+ "epoch": 10.140845070422536,
2562
+ "grad_norm": 0.2382764499634006,
2563
+ "learning_rate": 9.117355872777477e-07,
2564
+ "loss": 0.23789361119270325,
2565
+ "step": 365
2566
+ },
2567
+ {
2568
+ "epoch": 10.169014084507042,
2569
+ "grad_norm": 0.23857586756321417,
2570
+ "learning_rate": 9.112678638360015e-07,
2571
+ "loss": 0.2322252094745636,
2572
+ "step": 366
2573
+ },
2574
+ {
2575
+ "epoch": 10.19718309859155,
2576
+ "grad_norm": 0.24330523331980736,
2577
+ "learning_rate": 9.107990399096893e-07,
2578
+ "loss": 0.2315920889377594,
2579
+ "step": 367
2580
+ },
2581
+ {
2582
+ "epoch": 10.225352112676056,
2583
+ "grad_norm": 0.24433633894014556,
2584
+ "learning_rate": 9.103291169269299e-07,
2585
+ "loss": 0.2451440989971161,
2586
+ "step": 368
2587
+ },
2588
+ {
2589
+ "epoch": 10.253521126760564,
2590
+ "grad_norm": 0.24610806276364772,
2591
+ "learning_rate": 9.098580963191907e-07,
2592
+ "loss": 0.23949627578258514,
2593
+ "step": 369
2594
+ },
2595
+ {
2596
+ "epoch": 10.28169014084507,
2597
+ "grad_norm": 0.2523755182308413,
2598
+ "learning_rate": 9.093859795212817e-07,
2599
+ "loss": 0.255041241645813,
2600
+ "step": 370
2601
+ },
2602
+ {
2603
+ "epoch": 10.309859154929578,
2604
+ "grad_norm": 0.26254221360687513,
2605
+ "learning_rate": 9.089127679713529e-07,
2606
+ "loss": 0.24673743546009064,
2607
+ "step": 371
2608
+ },
2609
+ {
2610
+ "epoch": 10.338028169014084,
2611
+ "grad_norm": 0.2659551535025712,
2612
+ "learning_rate": 9.084384631108882e-07,
2613
+ "loss": 0.24092882871627808,
2614
+ "step": 372
2615
+ },
2616
+ {
2617
+ "epoch": 10.366197183098592,
2618
+ "grad_norm": 0.2498699583878038,
2619
+ "learning_rate": 9.079630663847031e-07,
2620
+ "loss": 0.25087377429008484,
2621
+ "step": 373
2622
+ },
2623
+ {
2624
+ "epoch": 10.394366197183098,
2625
+ "grad_norm": 0.24152932921989356,
2626
+ "learning_rate": 9.074865792409381e-07,
2627
+ "loss": 0.24652892351150513,
2628
+ "step": 374
2629
+ },
2630
+ {
2631
+ "epoch": 10.422535211267606,
2632
+ "grad_norm": 0.2975373111612133,
2633
+ "learning_rate": 9.070090031310558e-07,
2634
+ "loss": 0.24474310874938965,
2635
+ "step": 375
2636
+ },
2637
+ {
2638
+ "epoch": 10.450704225352112,
2639
+ "grad_norm": 0.24529378455418052,
2640
+ "learning_rate": 9.065303395098358e-07,
2641
+ "loss": 0.24984315037727356,
2642
+ "step": 376
2643
+ },
2644
+ {
2645
+ "epoch": 10.47887323943662,
2646
+ "grad_norm": 0.28844340433996196,
2647
+ "learning_rate": 9.060505898353705e-07,
2648
+ "loss": 0.2597285509109497,
2649
+ "step": 377
2650
+ },
2651
+ {
2652
+ "epoch": 10.507042253521126,
2653
+ "grad_norm": 0.2519686815558072,
2654
+ "learning_rate": 9.055697555690607e-07,
2655
+ "loss": 0.24229422211647034,
2656
+ "step": 378
2657
+ },
2658
+ {
2659
+ "epoch": 10.535211267605634,
2660
+ "grad_norm": 0.2540659099568006,
2661
+ "learning_rate": 9.050878381756107e-07,
2662
+ "loss": 0.24451278150081635,
2663
+ "step": 379
2664
+ },
2665
+ {
2666
+ "epoch": 10.56338028169014,
2667
+ "grad_norm": 0.25284268194412596,
2668
+ "learning_rate": 9.046048391230247e-07,
2669
+ "loss": 0.232026606798172,
2670
+ "step": 380
2671
+ },
2672
+ {
2673
+ "epoch": 10.591549295774648,
2674
+ "grad_norm": 0.2509160048760181,
2675
+ "learning_rate": 9.041207598826017e-07,
2676
+ "loss": 0.2384566068649292,
2677
+ "step": 381
2678
+ },
2679
+ {
2680
+ "epoch": 10.619718309859154,
2681
+ "grad_norm": 0.35519399545629293,
2682
+ "learning_rate": 9.036356019289309e-07,
2683
+ "loss": 0.24372628331184387,
2684
+ "step": 382
2685
+ },
2686
+ {
2687
+ "epoch": 10.647887323943662,
2688
+ "grad_norm": 0.2439787106834223,
2689
+ "learning_rate": 9.031493667398872e-07,
2690
+ "loss": 0.23376773297786713,
2691
+ "step": 383
2692
+ },
2693
+ {
2694
+ "epoch": 10.676056338028168,
2695
+ "grad_norm": 0.25658992692397564,
2696
+ "learning_rate": 9.026620557966279e-07,
2697
+ "loss": 0.2388392984867096,
2698
+ "step": 384
2699
+ },
2700
+ {
2701
+ "epoch": 10.704225352112676,
2702
+ "grad_norm": 0.2491570641039132,
2703
+ "learning_rate": 9.021736705835862e-07,
2704
+ "loss": 0.2429148256778717,
2705
+ "step": 385
2706
+ },
2707
+ {
2708
+ "epoch": 10.732394366197184,
2709
+ "grad_norm": 0.32073931464521355,
2710
+ "learning_rate": 9.016842125884684e-07,
2711
+ "loss": 0.22875139117240906,
2712
+ "step": 386
2713
+ },
2714
+ {
2715
+ "epoch": 10.76056338028169,
2716
+ "grad_norm": 0.2502280989278628,
2717
+ "learning_rate": 9.011936833022484e-07,
2718
+ "loss": 0.23325741291046143,
2719
+ "step": 387
2720
+ },
2721
+ {
2722
+ "epoch": 10.788732394366198,
2723
+ "grad_norm": 0.7429272352122884,
2724
+ "learning_rate": 9.007020842191634e-07,
2725
+ "loss": 0.25491032004356384,
2726
+ "step": 388
2727
+ },
2728
+ {
2729
+ "epoch": 10.816901408450704,
2730
+ "grad_norm": 0.25366909205038285,
2731
+ "learning_rate": 9.002094168367095e-07,
2732
+ "loss": 0.25390559434890747,
2733
+ "step": 389
2734
+ },
2735
+ {
2736
+ "epoch": 10.845070422535212,
2737
+ "grad_norm": 0.2532418479840136,
2738
+ "learning_rate": 8.997156826556369e-07,
2739
+ "loss": 0.239566832780838,
2740
+ "step": 390
2741
+ },
2742
+ {
2743
+ "epoch": 10.873239436619718,
2744
+ "grad_norm": 0.24862594273960315,
2745
+ "learning_rate": 8.992208831799456e-07,
2746
+ "loss": 0.24101951718330383,
2747
+ "step": 391
2748
+ },
2749
+ {
2750
+ "epoch": 10.901408450704226,
2751
+ "grad_norm": 0.36453597595458426,
2752
+ "learning_rate": 8.987250199168808e-07,
2753
+ "loss": 0.22447983920574188,
2754
+ "step": 392
2755
+ },
2756
+ {
2757
+ "epoch": 10.929577464788732,
2758
+ "grad_norm": 0.26909996327806845,
2759
+ "learning_rate": 8.982280943769278e-07,
2760
+ "loss": 0.24062535166740417,
2761
+ "step": 393
2762
+ },
2763
+ {
2764
+ "epoch": 10.95774647887324,
2765
+ "grad_norm": 0.2638850341939346,
2766
+ "learning_rate": 8.977301080738079e-07,
2767
+ "loss": 0.2585245966911316,
2768
+ "step": 394
2769
+ },
2770
+ {
2771
+ "epoch": 10.985915492957746,
2772
+ "grad_norm": 0.3097784840969631,
2773
+ "learning_rate": 8.97231062524474e-07,
2774
+ "loss": 0.2358318418264389,
2775
+ "step": 395
2776
+ },
2777
+ {
2778
+ "epoch": 11.0,
2779
+ "grad_norm": 0.3849386810629798,
2780
+ "learning_rate": 8.967309592491052e-07,
2781
+ "loss": 0.25533783435821533,
2782
+ "step": 396
2783
+ },
2784
+ {
2785
+ "epoch": 11.028169014084508,
2786
+ "grad_norm": 0.26256476010695584,
2787
+ "learning_rate": 8.962297997711027e-07,
2788
+ "loss": 0.22924092411994934,
2789
+ "step": 397
2790
+ },
2791
+ {
2792
+ "epoch": 11.056338028169014,
2793
+ "grad_norm": 0.2639241753105924,
2794
+ "learning_rate": 8.957275856170855e-07,
2795
+ "loss": 0.24100372195243835,
2796
+ "step": 398
2797
+ },
2798
+ {
2799
+ "epoch": 11.084507042253522,
2800
+ "grad_norm": 0.298152727306937,
2801
+ "learning_rate": 8.952243183168848e-07,
2802
+ "loss": 0.24404382705688477,
2803
+ "step": 399
2804
+ },
2805
+ {
2806
+ "epoch": 11.112676056338028,
2807
+ "grad_norm": 0.2702268162010949,
2808
+ "learning_rate": 8.9471999940354e-07,
2809
+ "loss": 0.23476135730743408,
2810
+ "step": 400
2811
+ },
2812
+ {
2813
+ "epoch": 11.140845070422536,
2814
+ "grad_norm": 0.38346237984716247,
2815
+ "learning_rate": 8.942146304132943e-07,
2816
+ "loss": 0.2230163812637329,
2817
+ "step": 401
2818
+ },
2819
+ {
2820
+ "epoch": 11.169014084507042,
2821
+ "grad_norm": 0.30719276755462255,
2822
+ "learning_rate": 8.937082128855891e-07,
2823
+ "loss": 0.24640659987926483,
2824
+ "step": 402
2825
+ },
2826
+ {
2827
+ "epoch": 11.19718309859155,
2828
+ "grad_norm": 0.2522559076259691,
2829
+ "learning_rate": 8.932007483630596e-07,
2830
+ "loss": 0.2337941825389862,
2831
+ "step": 403
2832
+ },
2833
+ {
2834
+ "epoch": 11.225352112676056,
2835
+ "grad_norm": 0.2811477850391036,
2836
+ "learning_rate": 8.926922383915315e-07,
2837
+ "loss": 0.24217446148395538,
2838
+ "step": 404
2839
+ },
2840
+ {
2841
+ "epoch": 11.253521126760564,
2842
+ "grad_norm": 0.2937577324495388,
2843
+ "learning_rate": 8.921826845200138e-07,
2844
+ "loss": 0.23650123178958893,
2845
+ "step": 405
2846
+ },
2847
+ {
2848
+ "epoch": 11.28169014084507,
2849
+ "grad_norm": 0.2665251490542768,
2850
+ "learning_rate": 8.916720883006963e-07,
2851
+ "loss": 0.23723018169403076,
2852
+ "step": 406
2853
+ },
2854
+ {
2855
+ "epoch": 11.309859154929578,
2856
+ "grad_norm": 0.2577956520635811,
2857
+ "learning_rate": 8.911604512889434e-07,
2858
+ "loss": 0.23821580410003662,
2859
+ "step": 407
2860
+ },
2861
+ {
2862
+ "epoch": 11.338028169014084,
2863
+ "grad_norm": 0.2747361963021525,
2864
+ "learning_rate": 8.906477750432903e-07,
2865
+ "loss": 0.23665378987789154,
2866
+ "step": 408
2867
+ },
2868
+ {
2869
+ "epoch": 11.366197183098592,
2870
+ "grad_norm": 0.36401043696800917,
2871
+ "learning_rate": 8.901340611254378e-07,
2872
+ "loss": 0.24189358949661255,
2873
+ "step": 409
2874
+ },
2875
+ {
2876
+ "epoch": 11.394366197183098,
2877
+ "grad_norm": 0.2520809121047697,
2878
+ "learning_rate": 8.896193111002475e-07,
2879
+ "loss": 0.2461044192314148,
2880
+ "step": 410
2881
+ },
2882
+ {
2883
+ "epoch": 11.422535211267606,
2884
+ "grad_norm": 0.2762514295319336,
2885
+ "learning_rate": 8.891035265357371e-07,
2886
+ "loss": 0.22839877009391785,
2887
+ "step": 411
2888
+ },
2889
+ {
2890
+ "epoch": 11.450704225352112,
2891
+ "grad_norm": 0.2766703038586183,
2892
+ "learning_rate": 8.88586709003076e-07,
2893
+ "loss": 0.2436380237340927,
2894
+ "step": 412
2895
+ },
2896
+ {
2897
+ "epoch": 11.47887323943662,
2898
+ "grad_norm": 0.25607243680163955,
2899
+ "learning_rate": 8.8806886007658e-07,
2900
+ "loss": 0.24581144750118256,
2901
+ "step": 413
2902
+ },
2903
+ {
2904
+ "epoch": 11.507042253521126,
2905
+ "grad_norm": 0.25612907513591854,
2906
+ "learning_rate": 8.875499813337067e-07,
2907
+ "loss": 0.24154463410377502,
2908
+ "step": 414
2909
+ },
2910
+ {
2911
+ "epoch": 11.535211267605634,
2912
+ "grad_norm": 0.26351210813960735,
2913
+ "learning_rate": 8.87030074355051e-07,
2914
+ "loss": 0.23528064787387848,
2915
+ "step": 415
2916
+ },
2917
+ {
2918
+ "epoch": 11.56338028169014,
2919
+ "grad_norm": 0.25456237789297487,
2920
+ "learning_rate": 8.865091407243394e-07,
2921
+ "loss": 0.23557059466838837,
2922
+ "step": 416
2923
+ },
2924
+ {
2925
+ "epoch": 11.591549295774648,
2926
+ "grad_norm": 0.26763665810579157,
2927
+ "learning_rate": 8.859871820284261e-07,
2928
+ "loss": 0.2497173398733139,
2929
+ "step": 417
2930
+ },
2931
+ {
2932
+ "epoch": 11.619718309859154,
2933
+ "grad_norm": 0.47514856857684123,
2934
+ "learning_rate": 8.85464199857288e-07,
2935
+ "loss": 0.24749311804771423,
2936
+ "step": 418
2937
+ },
2938
+ {
2939
+ "epoch": 11.647887323943662,
2940
+ "grad_norm": 0.5410123563974302,
2941
+ "learning_rate": 8.849401958040192e-07,
2942
+ "loss": 0.2356778383255005,
2943
+ "step": 419
2944
+ },
2945
+ {
2946
+ "epoch": 11.676056338028168,
2947
+ "grad_norm": 0.2597674855652335,
2948
+ "learning_rate": 8.844151714648274e-07,
2949
+ "loss": 0.24779964983463287,
2950
+ "step": 420
2951
+ },
2952
+ {
2953
+ "epoch": 11.704225352112676,
2954
+ "grad_norm": 0.26052872396427884,
2955
+ "learning_rate": 8.838891284390273e-07,
2956
+ "loss": 0.24069969356060028,
2957
+ "step": 421
2958
+ },
2959
+ {
2960
+ "epoch": 11.732394366197184,
2961
+ "grad_norm": 0.24768122719136823,
2962
+ "learning_rate": 8.833620683290375e-07,
2963
+ "loss": 0.23677203059196472,
2964
+ "step": 422
2965
+ },
2966
+ {
2967
+ "epoch": 11.76056338028169,
2968
+ "grad_norm": 0.2673480579083074,
2969
+ "learning_rate": 8.828339927403745e-07,
2970
+ "loss": 0.23263472318649292,
2971
+ "step": 423
2972
+ },
2973
+ {
2974
+ "epoch": 11.788732394366198,
2975
+ "grad_norm": 0.2542699067266434,
2976
+ "learning_rate": 8.823049032816478e-07,
2977
+ "loss": 0.23910850286483765,
2978
+ "step": 424
2979
+ },
2980
+ {
2981
+ "epoch": 11.816901408450704,
2982
+ "grad_norm": 0.2623369584647608,
2983
+ "learning_rate": 8.817748015645558e-07,
2984
+ "loss": 0.23500274121761322,
2985
+ "step": 425
2986
+ },
2987
+ {
2988
+ "epoch": 11.845070422535212,
2989
+ "grad_norm": 0.2741171560352947,
2990
+ "learning_rate": 8.812436892038805e-07,
2991
+ "loss": 0.2363435924053192,
2992
+ "step": 426
2993
+ },
2994
+ {
2995
+ "epoch": 11.873239436619718,
2996
+ "grad_norm": 0.2764265991901865,
2997
+ "learning_rate": 8.807115678174819e-07,
2998
+ "loss": 0.23622967302799225,
2999
+ "step": 427
3000
+ },
3001
+ {
3002
+ "epoch": 11.901408450704226,
3003
+ "grad_norm": 0.26408384561026765,
3004
+ "learning_rate": 8.801784390262943e-07,
3005
+ "loss": 0.2463335543870926,
3006
+ "step": 428
3007
+ },
3008
+ {
3009
+ "epoch": 11.929577464788732,
3010
+ "grad_norm": 0.2675833299403756,
3011
+ "learning_rate": 8.796443044543203e-07,
3012
+ "loss": 0.23592045903205872,
3013
+ "step": 429
3014
+ },
3015
+ {
3016
+ "epoch": 11.95774647887324,
3017
+ "grad_norm": 0.2992956465951693,
3018
+ "learning_rate": 8.791091657286267e-07,
3019
+ "loss": 0.23169052600860596,
3020
+ "step": 430
3021
+ },
3022
+ {
3023
+ "epoch": 11.985915492957746,
3024
+ "grad_norm": 0.25981524018584473,
3025
+ "learning_rate": 8.785730244793386e-07,
3026
+ "loss": 0.2329520881175995,
3027
+ "step": 431
3028
+ },
3029
+ {
3030
+ "epoch": 12.0,
3031
+ "grad_norm": 0.3813833592995728,
3032
+ "learning_rate": 8.780358823396352e-07,
3033
+ "loss": 0.24386751651763916,
3034
+ "step": 432
3035
+ }
3036
+ ],
3037
+ "logging_steps": 1,
3038
+ "max_steps": 1800,
3039
+ "num_input_tokens_seen": 0,
3040
+ "num_train_epochs": 50,
3041
+ "save_steps": 1.0,
3042
+ "stateful_callbacks": {
3043
+ "TrainerControl": {
3044
+ "args": {
3045
+ "should_epoch_stop": false,
3046
+ "should_evaluate": false,
3047
+ "should_log": false,
3048
+ "should_save": true,
3049
+ "should_training_stop": false
3050
+ },
3051
+ "attributes": {}
3052
+ }
3053
+ },
3054
+ "total_flos": 261065740386304.0,
3055
+ "train_batch_size": 16,
3056
+ "trial_name": null,
3057
+ "trial_params": null
3058
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99f7d1d2dee2896510b7f79d2f5f71441b9cbb37c7e9e71109da36e19612b2bc
3
+ size 9617
vocab.json ADDED
The diff for this file is too large to render. See raw diff
 
zero_to_fp32.py ADDED
@@ -0,0 +1,760 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example:
14
+ # python zero_to_fp32.py . output_dir/
15
+ # or
16
+ # python zero_to_fp32.py . output_dir/ --safe_serialization
17
+
18
+ import argparse
19
+ import torch
20
+ import glob
21
+ import math
22
+ import os
23
+ import re
24
+ import gc
25
+ import json
26
+ import numpy as np
27
+ from tqdm import tqdm
28
+ from collections import OrderedDict
29
+ from dataclasses import dataclass
30
+
31
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
32
+ # DeepSpeed data structures it has to be available in the current python environment.
33
+ from deepspeed.utils import logger
34
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
35
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
36
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
37
+
38
+
39
+ @dataclass
40
+ class zero_model_state:
41
+ buffers: dict()
42
+ param_shapes: dict()
43
+ shared_params: list
44
+ ds_version: int
45
+ frozen_param_shapes: dict()
46
+ frozen_param_fragments: dict()
47
+
48
+
49
+ debug = 0
50
+
51
+ # load to cpu
52
+ device = torch.device('cpu')
53
+
54
+
55
+ def atoi(text):
56
+ return int(text) if text.isdigit() else text
57
+
58
+
59
+ def natural_keys(text):
60
+ '''
61
+ alist.sort(key=natural_keys) sorts in human order
62
+ http://nedbatchelder.com/blog/200712/human_sorting.html
63
+ (See Toothy's implementation in the comments)
64
+ '''
65
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
66
+
67
+
68
+ def get_model_state_file(checkpoint_dir, zero_stage):
69
+ if not os.path.isdir(checkpoint_dir):
70
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
71
+
72
+ # there should be only one file
73
+ if zero_stage <= 2:
74
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
75
+ elif zero_stage == 3:
76
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
77
+
78
+ if not os.path.exists(file):
79
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
80
+
81
+ return file
82
+
83
+
84
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
85
+ # XXX: need to test that this simple glob rule works for multi-node setup too
86
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
87
+
88
+ if len(ckpt_files) == 0:
89
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
90
+
91
+ return ckpt_files
92
+
93
+
94
+ def get_optim_files(checkpoint_dir):
95
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
96
+
97
+
98
+ def get_model_state_files(checkpoint_dir):
99
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
100
+
101
+
102
+ def parse_model_states(files):
103
+ zero_model_states = []
104
+ for file in files:
105
+ state_dict = torch.load(file, map_location=device, weights_only=False)
106
+
107
+ if BUFFER_NAMES not in state_dict:
108
+ raise ValueError(f"{file} is not a model state checkpoint")
109
+ buffer_names = state_dict[BUFFER_NAMES]
110
+ if debug:
111
+ print("Found buffers:", buffer_names)
112
+
113
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
114
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
115
+ param_shapes = state_dict[PARAM_SHAPES]
116
+
117
+ # collect parameters that are included in param_shapes
118
+ param_names = []
119
+ for s in param_shapes:
120
+ for name in s.keys():
121
+ param_names.append(name)
122
+
123
+ # update with frozen parameters
124
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
125
+ if frozen_param_shapes is not None:
126
+ if debug:
127
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
128
+ param_names += list(frozen_param_shapes.keys())
129
+
130
+ # handle shared params
131
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
132
+
133
+ ds_version = state_dict.get(DS_VERSION, None)
134
+
135
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
136
+
137
+ z_model_state = zero_model_state(buffers=buffers,
138
+ param_shapes=param_shapes,
139
+ shared_params=shared_params,
140
+ ds_version=ds_version,
141
+ frozen_param_shapes=frozen_param_shapes,
142
+ frozen_param_fragments=frozen_param_fragments)
143
+ zero_model_states.append(z_model_state)
144
+
145
+ return zero_model_states
146
+
147
+
148
+ def parse_optim_states(files, ds_checkpoint_dir):
149
+ total_files = len(files)
150
+ state_dicts = []
151
+ for f in tqdm(files, desc='Loading checkpoint shards'):
152
+ state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
153
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
154
+ # and also handle the case where it was already removed by another helper script
155
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
156
+ state_dicts.append(state_dict)
157
+
158
+ if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
159
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
160
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
161
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
162
+
163
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
164
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
165
+ # use the max of the partition_count to get the dp world_size.
166
+
167
+ if type(world_size) is list:
168
+ world_size = max(world_size)
169
+
170
+ if world_size != total_files:
171
+ raise ValueError(
172
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
173
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
174
+ )
175
+
176
+ # the groups are named differently in each stage
177
+ if zero_stage <= 2:
178
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
179
+ elif zero_stage == 3:
180
+ fp32_groups_key = FP32_FLAT_GROUPS
181
+ else:
182
+ raise ValueError(f"unknown zero stage {zero_stage}")
183
+
184
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
185
+ return zero_stage, world_size, fp32_flat_groups
186
+
187
+
188
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
189
+ """
190
+ Returns fp32 state_dict reconstructed from ds checkpoint
191
+
192
+ Args:
193
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
194
+
195
+ """
196
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
197
+
198
+ optim_files = get_optim_files(ds_checkpoint_dir)
199
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
200
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
201
+
202
+ model_files = get_model_state_files(ds_checkpoint_dir)
203
+
204
+ zero_model_states = parse_model_states(model_files)
205
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
206
+
207
+ if zero_stage <= 2:
208
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
209
+ exclude_frozen_parameters)
210
+ elif zero_stage == 3:
211
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
212
+ exclude_frozen_parameters)
213
+
214
+
215
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
216
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
217
+ return
218
+
219
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
220
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
221
+
222
+ if debug:
223
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
224
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
225
+
226
+ wanted_params = len(frozen_param_shapes)
227
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
228
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
229
+ print(f'Frozen params: Have {avail_numel} numels to process.')
230
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
231
+
232
+ total_params = 0
233
+ total_numel = 0
234
+ for name, shape in frozen_param_shapes.items():
235
+ total_params += 1
236
+ unpartitioned_numel = shape.numel()
237
+ total_numel += unpartitioned_numel
238
+
239
+ state_dict[name] = frozen_param_fragments[name]
240
+
241
+ if debug:
242
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
243
+
244
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
245
+
246
+
247
+ def _has_callable(obj, fn):
248
+ attr = getattr(obj, fn, None)
249
+ return callable(attr)
250
+
251
+
252
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
253
+ param_shapes = zero_model_states[0].param_shapes
254
+
255
+ # Reconstruction protocol:
256
+ #
257
+ # XXX: document this
258
+
259
+ if debug:
260
+ for i in range(world_size):
261
+ for j in range(len(fp32_flat_groups[0])):
262
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
263
+
264
+ # XXX: memory usage doubles here (zero2)
265
+ num_param_groups = len(fp32_flat_groups[0])
266
+ merged_single_partition_of_fp32_groups = []
267
+ for i in range(num_param_groups):
268
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
269
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
270
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
271
+ avail_numel = sum(
272
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
273
+
274
+ if debug:
275
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
276
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
277
+ # not asserting if there is a mismatch due to possible padding
278
+ print(f"Have {avail_numel} numels to process.")
279
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
280
+
281
+ # params
282
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
283
+ # out-of-core computing solution
284
+ total_numel = 0
285
+ total_params = 0
286
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
287
+ offset = 0
288
+ avail_numel = full_single_fp32_vector.numel()
289
+ for name, shape in shapes.items():
290
+
291
+ unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
292
+ total_numel += unpartitioned_numel
293
+ total_params += 1
294
+
295
+ if debug:
296
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
297
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
298
+ offset += unpartitioned_numel
299
+
300
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
301
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
302
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
303
+ # live optimizer object, so we are checking that the numbers are within the right range
304
+ align_to = 2 * world_size
305
+
306
+ def zero2_align(x):
307
+ return align_to * math.ceil(x / align_to)
308
+
309
+ if debug:
310
+ print(f"original offset={offset}, avail_numel={avail_numel}")
311
+
312
+ offset = zero2_align(offset)
313
+ avail_numel = zero2_align(avail_numel)
314
+
315
+ if debug:
316
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
317
+
318
+ # Sanity check
319
+ if offset != avail_numel:
320
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
321
+
322
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
323
+
324
+
325
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
326
+ exclude_frozen_parameters):
327
+ state_dict = OrderedDict()
328
+
329
+ # buffers
330
+ buffers = zero_model_states[0].buffers
331
+ state_dict.update(buffers)
332
+ if debug:
333
+ print(f"added {len(buffers)} buffers")
334
+
335
+ if not exclude_frozen_parameters:
336
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
337
+
338
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
339
+
340
+ # recover shared parameters
341
+ for pair in zero_model_states[0].shared_params:
342
+ if pair[1] in state_dict:
343
+ state_dict[pair[0]] = state_dict[pair[1]]
344
+
345
+ return state_dict
346
+
347
+
348
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
349
+ remainder = unpartitioned_numel % world_size
350
+ padding_numel = (world_size - remainder) if remainder else 0
351
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
352
+ return partitioned_numel, padding_numel
353
+
354
+
355
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
356
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
357
+ return
358
+
359
+ if debug:
360
+ for i in range(world_size):
361
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
362
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
363
+
364
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
365
+ wanted_params = len(frozen_param_shapes)
366
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
367
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
368
+ print(f'Frozen params: Have {avail_numel} numels to process.')
369
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
370
+
371
+ total_params = 0
372
+ total_numel = 0
373
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
374
+ total_params += 1
375
+ unpartitioned_numel = shape.numel()
376
+ total_numel += unpartitioned_numel
377
+
378
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
379
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
380
+
381
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
382
+
383
+ if debug:
384
+ print(
385
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
386
+ )
387
+
388
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
389
+
390
+
391
+ class GatheredTensor:
392
+ """
393
+ A pseudo tensor that collects partitioned weights.
394
+ It is more memory efficient when there are multiple groups.
395
+ """
396
+
397
+ def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
398
+ self.flat_groups = flat_groups
399
+ self.flat_groups_offset = flat_groups_offset
400
+ self.offset = offset
401
+ self.partitioned_numel = partitioned_numel
402
+ self.shape = shape
403
+ self.dtype = self.flat_groups[0][0].dtype
404
+
405
+ def contiguous(self):
406
+ """
407
+ Merge partitioned weights from flat_groups into a single tensor.
408
+ """
409
+ end_idx = self.offset + self.partitioned_numel
410
+ world_size = len(self.flat_groups)
411
+ pad_flat_param_chunks = []
412
+
413
+ for rank_i in range(world_size):
414
+ # for each rank, we need to collect weights from related group/groups
415
+ flat_groups_at_rank_i = self.flat_groups[rank_i]
416
+ start_group_id = None
417
+ end_group_id = None
418
+ for group_id in range(len(self.flat_groups_offset)):
419
+ if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
420
+ start_group_id = group_id
421
+ if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
422
+ end_group_id = group_id
423
+ break
424
+ # collect weights from related group/groups
425
+ for group_id in range(start_group_id, end_group_id + 1):
426
+ flat_tensor = flat_groups_at_rank_i[group_id]
427
+ start_offset = self.offset - self.flat_groups_offset[group_id]
428
+ end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
429
+ pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
430
+
431
+ # collect weights from all ranks
432
+ pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
433
+ param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
434
+ return param
435
+
436
+
437
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
438
+ param_shapes = zero_model_states[0].param_shapes
439
+ avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
440
+
441
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
442
+ # param, re-consolidating each param, while dealing with padding if any
443
+
444
+ # merge list of dicts, preserving order
445
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
446
+
447
+ if debug:
448
+ for i in range(world_size):
449
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
450
+
451
+ wanted_params = len(param_shapes)
452
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
453
+ # not asserting if there is a mismatch due to possible padding
454
+ avail_numel = fp32_flat_groups[0].numel() * world_size
455
+ print(f"Trainable params: Have {avail_numel} numels to process.")
456
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
457
+
458
+ # params
459
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
460
+ # out-of-core computing solution
461
+ offset = 0
462
+ total_numel = 0
463
+ total_params = 0
464
+ flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
465
+ for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
466
+ unpartitioned_numel = shape.numel()
467
+ total_numel += unpartitioned_numel
468
+ total_params += 1
469
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
470
+
471
+ if debug:
472
+ print(
473
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
474
+ )
475
+
476
+ # memory efficient tensor
477
+ tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
478
+ state_dict[name] = tensor
479
+ offset += partitioned_numel
480
+
481
+ offset *= world_size
482
+
483
+ # Sanity check
484
+ if offset != avail_numel:
485
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
486
+
487
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
488
+
489
+
490
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
491
+ exclude_frozen_parameters):
492
+ state_dict = OrderedDict()
493
+
494
+ # buffers
495
+ buffers = zero_model_states[0].buffers
496
+ state_dict.update(buffers)
497
+ if debug:
498
+ print(f"added {len(buffers)} buffers")
499
+
500
+ if not exclude_frozen_parameters:
501
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
502
+
503
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
504
+
505
+ # recover shared parameters
506
+ for pair in zero_model_states[0].shared_params:
507
+ if pair[1] in state_dict:
508
+ state_dict[pair[0]] = state_dict[pair[1]]
509
+
510
+ return state_dict
511
+
512
+
513
+ def to_torch_tensor(state_dict, return_empty_tensor=False):
514
+ """
515
+ Convert state_dict of GatheredTensor to torch tensor
516
+ """
517
+ torch_state_dict = {}
518
+ converted_tensors = {}
519
+ for name, tensor in state_dict.items():
520
+ tensor_id = id(tensor)
521
+ if tensor_id in converted_tensors: # shared tensors
522
+ shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
523
+ torch_state_dict[name] = shared_tensor
524
+ else:
525
+ converted_tensors[tensor_id] = name
526
+ if return_empty_tensor:
527
+ torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
528
+ else:
529
+ torch_state_dict[name] = tensor.contiguous()
530
+ return torch_state_dict
531
+
532
+
533
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
534
+ tag=None,
535
+ exclude_frozen_parameters=False,
536
+ lazy_mode=False):
537
+ """
538
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
539
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
540
+ via a model hub.
541
+
542
+ Args:
543
+ - ``checkpoint_dir``: path to the desired checkpoint folder
544
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
545
+ - ``exclude_frozen_parameters``: exclude frozen parameters
546
+ - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
547
+ Convert the pesduo tensor to torch tensor by ``.contiguous()``
548
+
549
+ Returns:
550
+ - pytorch ``state_dict``
551
+
552
+ A typical usage might be ::
553
+
554
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
555
+ # do the training and checkpoint saving
556
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
557
+ model = model.cpu() # move to cpu
558
+ model.load_state_dict(state_dict)
559
+ # submit to model hub or save the model to share with others
560
+
561
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
562
+ application. i.e. you will need to re-initialize the deepspeed engine, since
563
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
564
+
565
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
566
+
567
+ Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
568
+ You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
569
+ the checkpoint. Or you can load state_dict in lazy mode ::
570
+
571
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
572
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
573
+ for name, lazy_tensor in state_dict.item():
574
+ tensor = lazy_tensor.contiguous() # to cpu
575
+ print(name, tensor)
576
+ # del tensor to release memory if it no longer in use
577
+ """
578
+ if tag is None:
579
+ latest_path = os.path.join(checkpoint_dir, 'latest')
580
+ if os.path.isfile(latest_path):
581
+ with open(latest_path, 'r') as fd:
582
+ tag = fd.read().strip()
583
+ else:
584
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
585
+
586
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
587
+
588
+ if not os.path.isdir(ds_checkpoint_dir):
589
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
590
+
591
+ state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
592
+ if lazy_mode:
593
+ return state_dict
594
+ else:
595
+ return to_torch_tensor(state_dict)
596
+
597
+
598
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
599
+ output_dir,
600
+ max_shard_size="5GB",
601
+ safe_serialization=False,
602
+ tag=None,
603
+ exclude_frozen_parameters=False):
604
+ """
605
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
606
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
607
+
608
+ Args:
609
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
610
+ - ``output_dir``: directory to the pytorch fp32 state_dict output files
611
+ - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
612
+ - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
613
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
614
+ - ``exclude_frozen_parameters``: exclude frozen parameters
615
+ """
616
+
617
+ # Dependency pre-check
618
+ if safe_serialization:
619
+ try:
620
+ from safetensors.torch import save_file
621
+ except ImportError:
622
+ print('If you want to use `safe_serialization`, please `pip install safetensors`')
623
+ raise
624
+ if max_shard_size is not None:
625
+ try:
626
+ from huggingface_hub import split_torch_state_dict_into_shards
627
+ except ImportError:
628
+ print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
629
+ raise
630
+
631
+ # Convert zero checkpoint to state_dict
632
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
633
+ tag,
634
+ exclude_frozen_parameters,
635
+ lazy_mode=True)
636
+
637
+ # Shard the model if it is too big.
638
+ weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
639
+ if max_shard_size is not None:
640
+ filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
641
+ # an memory-efficient approach for sharding
642
+ empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
643
+ state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
644
+ filename_pattern=filename_pattern,
645
+ max_shard_size=max_shard_size)
646
+ else:
647
+ from collections import namedtuple
648
+ StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
649
+ state_dict_split = StateDictSplit(is_sharded=False,
650
+ filename_to_tensors={weights_name: list(state_dict.keys())})
651
+
652
+ # Save the model by shard
653
+ os.makedirs(output_dir, exist_ok=True)
654
+ filename_to_tensors = state_dict_split.filename_to_tensors.items()
655
+ for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
656
+ shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
657
+ shard_state_dict = to_torch_tensor(shard_state_dict)
658
+ output_path = os.path.join(output_dir, shard_file)
659
+ if safe_serialization:
660
+ save_file(shard_state_dict, output_path, metadata={"format": "pt"})
661
+ else:
662
+ torch.save(shard_state_dict, output_path)
663
+ # release the memory of current shard
664
+ for tensor_name in list(shard_state_dict.keys()):
665
+ del state_dict[tensor_name]
666
+ del shard_state_dict[tensor_name]
667
+ del shard_state_dict
668
+ gc.collect()
669
+
670
+ # Save index if sharded
671
+ if state_dict_split.is_sharded:
672
+ index = {
673
+ "metadata": state_dict_split.metadata,
674
+ "weight_map": state_dict_split.tensor_to_filename,
675
+ }
676
+ save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
677
+ save_index_file = os.path.join(output_dir, save_index_file)
678
+ with open(save_index_file, "w", encoding="utf-8") as f:
679
+ content = json.dumps(index, indent=2, sort_keys=True) + "\n"
680
+ f.write(content)
681
+
682
+
683
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
684
+ """
685
+ 1. Put the provided model to cpu
686
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
687
+ 3. Load it into the provided model
688
+
689
+ Args:
690
+ - ``model``: the model object to update
691
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
692
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
693
+
694
+ Returns:
695
+ - ``model`: modified model
696
+
697
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
698
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
699
+ conveniently placed for you in the checkpoint folder.
700
+
701
+ A typical usage might be ::
702
+
703
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
704
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
705
+ # submit to model hub or save the model to share with others
706
+
707
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
708
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
709
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
710
+
711
+ """
712
+ logger.info("Extracting fp32 weights")
713
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
714
+
715
+ logger.info("Overwriting model with fp32 weights")
716
+ model = model.cpu()
717
+ model.load_state_dict(state_dict, strict=False)
718
+
719
+ return model
720
+
721
+
722
+ if __name__ == "__main__":
723
+ parser = argparse.ArgumentParser()
724
+ parser.add_argument("checkpoint_dir",
725
+ type=str,
726
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
727
+ parser.add_argument("output_dir",
728
+ type=str,
729
+ help="directory to the pytorch fp32 state_dict output files"
730
+ "(e.g. path/checkpoint-12-output/)")
731
+ parser.add_argument(
732
+ "--max_shard_size",
733
+ type=str,
734
+ default="5GB",
735
+ help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
736
+ "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
737
+ "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
738
+ "without CPU OOM issues.")
739
+ parser.add_argument(
740
+ "--safe_serialization",
741
+ default=False,
742
+ action='store_true',
743
+ help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
744
+ parser.add_argument("-t",
745
+ "--tag",
746
+ type=str,
747
+ default=None,
748
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
749
+ parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
750
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
751
+ args = parser.parse_args()
752
+
753
+ debug = args.debug
754
+
755
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
756
+ args.output_dir,
757
+ max_shard_size=args.max_shard_size,
758
+ safe_serialization=args.safe_serialization,
759
+ tag=args.tag,
760
+ exclude_frozen_parameters=args.exclude_frozen_parameters)