cogito233 commited on Oct 12, 2025

Commit

0f8c792

verified ·

1 Parent(s): e377f91

Init: upload full folder recursively

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +4 -0
v0-20251012-211828/args.json +369 -0
v0-20251012-211828/checkpoint-224/added_tokens.json +24 -0
v0-20251012-211828/checkpoint-224/args.json +369 -0
v0-20251012-211828/checkpoint-224/config.json +30 -0
v0-20251012-211828/checkpoint-224/generation_config.json +6 -0
v0-20251012-211828/checkpoint-224/latest +1 -0
v0-20251012-211828/checkpoint-224/merges.txt +0 -0
v0-20251012-211828/checkpoint-224/model-00001-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-224/model-00002-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-224/model-00003-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-224/model-00004-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-224/model.safetensors.index.json +346 -0
v0-20251012-211828/checkpoint-224/rng_state_0.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_1.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_2.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_3.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_4.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_5.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_6.pth +3 -0
v0-20251012-211828/checkpoint-224/rng_state_7.pth +3 -0
v0-20251012-211828/checkpoint-224/scheduler.pt +3 -0
v0-20251012-211828/checkpoint-224/special_tokens_map.json +31 -0
v0-20251012-211828/checkpoint-224/tokenizer.json +3 -0
v0-20251012-211828/checkpoint-224/tokenizer_config.json +208 -0
v0-20251012-211828/checkpoint-224/trainer_state.json +2283 -0
v0-20251012-211828/checkpoint-224/training_args.bin +3 -0
v0-20251012-211828/checkpoint-224/vocab.json +0 -0
v0-20251012-211828/checkpoint-224/zero_to_fp32.py +604 -0
v0-20251012-211828/checkpoint-448/added_tokens.json +24 -0
v0-20251012-211828/checkpoint-448/args.json +369 -0
v0-20251012-211828/checkpoint-448/config.json +30 -0
v0-20251012-211828/checkpoint-448/generation_config.json +6 -0
v0-20251012-211828/checkpoint-448/latest +1 -0
v0-20251012-211828/checkpoint-448/merges.txt +0 -0
v0-20251012-211828/checkpoint-448/model-00001-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-448/model-00002-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-448/model-00003-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-448/model-00004-of-00004.safetensors +3 -0
v0-20251012-211828/checkpoint-448/model.safetensors.index.json +346 -0
v0-20251012-211828/checkpoint-448/rng_state_0.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_1.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_2.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_3.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_4.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_5.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_6.pth +3 -0
v0-20251012-211828/checkpoint-448/rng_state_7.pth +3 -0
v0-20251012-211828/checkpoint-448/scheduler.pt +3 -0
v0-20251012-211828/checkpoint-448/special_tokens_map.json +31 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+v0-20251012-211828/checkpoint-224/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v0-20251012-211828/checkpoint-448/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v0-20251012-211828/checkpoint-672/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v0-20251012-211828/checkpoint-896/tokenizer.json filter=lfs diff=lfs merge=lfs -text

v0-20251012-211828/args.json ADDED Viewed

	@@ -0,0 +1,369 @@

+{
+  "model": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "model_type": "qwen2_5",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "response_prefix": null,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "use_chat_template": true,
+  "template_backend": "swift",
+  "dataset": [
+    "/data/minimax-dialogue/users/ruobai/cogito/base_dataset/broswer_training_data/merged_data.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "data_seed": 42,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": false,
+  "model_name": [
+    null,
+    null
+  ],
+  "model_author": [
+    null,
+    null
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": true,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ddp_timeout": 1800,
+  "ddp_backend": "nccl",
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 1e-05,
+  "weight_decay": 0.05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 4.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 20.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "wandb"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": false,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": true,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "optimizer": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 8,
+  "local_world_size": 8,
+  "model_suffix": "Qwen2.5-7B",
+  "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding'])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f3ff78a7920>, model_arch='llama', architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
+  "model_dir": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.05, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=4.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend='nccl', tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=20.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['wandb'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='full', optimizer=None, local_repo_path=None, galore_config=None)"
+}

v0-20251012-211828/checkpoint-224/added_tokens.json ADDED Viewed

	@@ -0,0 +1,24 @@

+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

v0-20251012-211828/checkpoint-224/args.json ADDED Viewed

	@@ -0,0 +1,369 @@

+{
+  "model": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "model_type": "qwen2_5",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "response_prefix": null,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "use_chat_template": true,
+  "template_backend": "swift",
+  "dataset": [
+    "/data/minimax-dialogue/users/ruobai/cogito/base_dataset/broswer_training_data/merged_data.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "data_seed": 42,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": false,
+  "model_name": [
+    null,
+    null
+  ],
+  "model_author": [
+    null,
+    null
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": true,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ddp_timeout": 1800,
+  "ddp_backend": "nccl",
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 1e-05,
+  "weight_decay": 0.05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 4.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 20.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "wandb"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": false,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": true,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "optimizer": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 8,
+  "local_world_size": 8,
+  "model_suffix": "Qwen2.5-7B",
+  "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding'])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f3ff78a7920>, model_arch='llama', architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
+  "model_dir": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.05, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=4.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend='nccl', tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=20.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['wandb'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='full', optimizer=None, local_repo_path=None, galore_config=None)"
+}

v0-20251012-211828/checkpoint-224/config.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

v0-20251012-211828/checkpoint-224/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.51.3"
+}

v0-20251012-211828/checkpoint-224/latest ADDED Viewed

	@@ -0,0 +1 @@


1	+ global_step224

v0-20251012-211828/checkpoint-224/merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

v0-20251012-211828/checkpoint-224/model-00001-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:47e4a50ad6f22212a03f72a90780581c18fc74c61f7a496b2843fca3bc07980f
+size 4877660776

v0-20251012-211828/checkpoint-224/model-00002-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:71449c6a8393462bda8cbb108ef372376ac9732605ef73025444a1152a1ae057
+size 4932751008

v0-20251012-211828/checkpoint-224/model-00003-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0967793d81e50b062ebcedbe59362c8c569ea38092559427069ce67f86b415c4
+size 4330865200

v0-20251012-211828/checkpoint-224/model-00004-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9faafb4490147e060ccd2871de2a1e2582060fce40dfb9c88329ec6f60348b46
+size 1089994880

v0-20251012-211828/checkpoint-224/model.safetensors.index.json ADDED Viewed

	@@ -0,0 +1,346 @@

+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

v0-20251012-211828/checkpoint-224/rng_state_0.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2946430e67d53bbdba29ad82934bbaaedc694d31b613744d3a4e1a1e9a46ba8f
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_1.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d2f8ffeab37684493e90f06d39e88dd23930f6d4b22434e761cf24eb66c33624
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_2.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f3e34f4b25d022859f36dc8565e9864ef89df31dee0b6f28ae330b4505432890
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_3.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2c75713cb224376ed36decc995102191568e3ae75807c8df22de6bdfa763f0e1
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_4.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:6a045b81eb8aff0c6f04e5c1c068c22b50a29b0c3cd0ceb0f7b0633e087645e2
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_5.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:4fbb9337a1f2ddc7522ae749e74101fdc6408581ff2447c6ce01d1264feaa554
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_6.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:bd4f55e2919a65fc83570ea3e656aee5c665014b2d4ebb1e757b6604bfa2f91e
+size 15920

v0-20251012-211828/checkpoint-224/rng_state_7.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e742eccb95f66543b690641ccc536fb0f2c7f6d2ad1dce55de41ec2f9bcac81a
+size 15920

v0-20251012-211828/checkpoint-224/scheduler.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:bbd6c9531096989d9e391789dabe461a76afc83be7dae88308783658168c9414
+size 1064

v0-20251012-211828/checkpoint-224/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

v0-20251012-211828/checkpoint-224/tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

v0-20251012-211828/checkpoint-224/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,208 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

v0-20251012-211828/checkpoint-224/trainer_state.json ADDED Viewed

	@@ -0,0 +1,2283 @@

+{
+  "best_global_step": 224,
+  "best_metric": 0.36971456,
+  "best_model_checkpoint": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/checkpoint-224",
+  "epoch": 1.0,
+  "eval_steps": 20.0,
+  "global_step": 224,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004464285714285714,
+      "grad_norm": 607.0161258405667,
+      "learning_rate": 2.2222222222222224e-07,
+      "loss": 1.2842669486999512,
+      "memory(GiB)": 73.22,
+      "step": 1,
+      "token_acc": 0.7433547204399633,
+      "train_speed(iter/s)": 0.074483
+    },
+    {
+      "epoch": 0.008928571428571428,
+      "grad_norm": 416.31840864440954,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 1.3240495920181274,
+      "memory(GiB)": 73.22,
+      "step": 2,
+      "token_acc": 0.6602722772277227,
+      "train_speed(iter/s)": 0.112796
+    },
+    {
+      "epoch": 0.013392857142857142,
+      "grad_norm": 160.17360140132547,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 1.2871371507644653,
+      "memory(GiB)": 73.22,
+      "step": 3,
+      "token_acc": 0.6800364630811303,
+      "train_speed(iter/s)": 0.136364
+    },
+    {
+      "epoch": 0.017857142857142856,
+      "grad_norm": 450.37534576513326,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 1.2711200714111328,
+      "memory(GiB)": 100.82,
+      "step": 4,
+      "token_acc": 0.7110352673492605,
+      "train_speed(iter/s)": 0.15202
+    },
+    {
+      "epoch": 0.022321428571428572,
+      "grad_norm": 442.68256941397544,
+      "learning_rate": 1.111111111111111e-06,
+      "loss": 1.2603538036346436,
+      "memory(GiB)": 100.82,
+      "step": 5,
+      "token_acc": 0.7108695652173913,
+      "train_speed(iter/s)": 0.161769
+    },
+    {
+      "epoch": 0.026785714285714284,
+      "grad_norm": 290.49695787962474,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 1.2085299491882324,
+      "memory(GiB)": 100.82,
+      "step": 6,
+      "token_acc": 0.7053445850914205,
+      "train_speed(iter/s)": 0.170606
+    },
+    {
+      "epoch": 0.03125,
+      "grad_norm": 122.03536728572118,
+      "learning_rate": 1.5555555555555558e-06,
+      "loss": 1.1946239471435547,
+      "memory(GiB)": 100.82,
+      "step": 7,
+      "token_acc": 0.7184265010351967,
+      "train_speed(iter/s)": 0.177314
+    },
+    {
+      "epoch": 0.03571428571428571,
+      "grad_norm": 48.184047801314975,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 1.1202031373977661,
+      "memory(GiB)": 100.82,
+      "step": 8,
+      "token_acc": 0.692090395480226,
+      "train_speed(iter/s)": 0.182678
+    },
+    {
+      "epoch": 0.04017857142857143,
+      "grad_norm": 64.05149006299645,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 1.058239459991455,
+      "memory(GiB)": 100.82,
+      "step": 9,
+      "token_acc": 0.735609756097561,
+      "train_speed(iter/s)": 0.186756
+    },
+    {
+      "epoch": 0.044642857142857144,
+      "grad_norm": 32.35823954189375,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 1.0513522624969482,
+      "memory(GiB)": 128.42,
+      "step": 10,
+      "token_acc": 0.7422945205479452,
+      "train_speed(iter/s)": 0.190564
+    },
+    {
+      "epoch": 0.049107142857142856,
+      "grad_norm": 33.455251112438354,
+      "learning_rate": 2.4444444444444447e-06,
+      "loss": 0.8908174633979797,
+      "memory(GiB)": 128.42,
+      "step": 11,
+      "token_acc": 0.7462585034013606,
+      "train_speed(iter/s)": 0.19371
+    },
+    {
+      "epoch": 0.05357142857142857,
+      "grad_norm": 20.237589968447807,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.9060617089271545,
+      "memory(GiB)": 128.42,
+      "step": 12,
+      "token_acc": 0.7580266249021144,
+      "train_speed(iter/s)": 0.196325
+    },
+    {
+      "epoch": 0.05803571428571429,
+      "grad_norm": 19.104830741854588,
+      "learning_rate": 2.888888888888889e-06,
+      "loss": 0.8370632529258728,
+      "memory(GiB)": 128.42,
+      "step": 13,
+      "token_acc": 0.7583454281567489,
+      "train_speed(iter/s)": 0.198756
+    },
+    {
+      "epoch": 0.0625,
+      "grad_norm": 8.07005348419444,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 0.7916728258132935,
+      "memory(GiB)": 128.42,
+      "step": 14,
+      "token_acc": 0.7912087912087912,
+      "train_speed(iter/s)": 0.200858
+    },
+    {
+      "epoch": 0.06696428571428571,
+      "grad_norm": 9.110633598653314,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.7699524164199829,
+      "memory(GiB)": 128.42,
+      "step": 15,
+      "token_acc": 0.7777777777777778,
+      "train_speed(iter/s)": 0.202712
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 5.340593386565067,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 0.7363409996032715,
+      "memory(GiB)": 128.42,
+      "step": 16,
+      "token_acc": 0.7858439201451906,
+      "train_speed(iter/s)": 0.204218
+    },
+    {
+      "epoch": 0.07589285714285714,
+      "grad_norm": 9.109649268071344,
+      "learning_rate": 3.777777777777778e-06,
+      "loss": 0.7368918657302856,
+      "memory(GiB)": 128.42,
+      "step": 17,
+      "token_acc": 0.7816216216216216,
+      "train_speed(iter/s)": 0.205652
+    },
+    {
+      "epoch": 0.08035714285714286,
+      "grad_norm": 11.131867979943577,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.6874697804450989,
+      "memory(GiB)": 128.42,
+      "step": 18,
+      "token_acc": 0.7817299919159256,
+      "train_speed(iter/s)": 0.20693
+    },
+    {
+      "epoch": 0.08482142857142858,
+      "grad_norm": 4.574035974729758,
+      "learning_rate": 4.222222222222223e-06,
+      "loss": 0.6693994998931885,
+      "memory(GiB)": 128.42,
+      "step": 19,
+      "token_acc": 0.8025659301496793,
+      "train_speed(iter/s)": 0.208049
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 4.387355536169233,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.6153415441513062,
+      "memory(GiB)": 128.42,
+      "step": 20,
+      "token_acc": 0.8143491124260355,
+      "train_speed(iter/s)": 0.20903
+    },
+    {
+      "epoch": 0.09375,
+      "grad_norm": 5.155423473676148,
+      "learning_rate": 4.666666666666667e-06,
+      "loss": 0.6413918137550354,
+      "memory(GiB)": 128.42,
+      "step": 21,
+      "token_acc": 0.8059581320450886,
+      "train_speed(iter/s)": 0.209872
+    },
+    {
+      "epoch": 0.09821428571428571,
+      "grad_norm": 4.52090458658663,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.6642692685127258,
+      "memory(GiB)": 128.42,
+      "step": 22,
+      "token_acc": 0.8458274398868458,
+      "train_speed(iter/s)": 0.210769
+    },
+    {
+      "epoch": 0.10267857142857142,
+      "grad_norm": 4.0676391904396105,
+      "learning_rate": 5.1111111111111115e-06,
+      "loss": 0.612501859664917,
+      "memory(GiB)": 128.42,
+      "step": 23,
+      "token_acc": 0.8329896907216495,
+      "train_speed(iter/s)": 0.211563
+    },
+    {
+      "epoch": 0.10714285714285714,
+      "grad_norm": 9.05605916321185,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 0.5991834402084351,
+      "memory(GiB)": 128.42,
+      "step": 24,
+      "token_acc": 0.820136852394917,
+      "train_speed(iter/s)": 0.212389
+    },
+    {
+      "epoch": 0.11160714285714286,
+      "grad_norm": 4.0651164644201945,
+      "learning_rate": 5.555555555555557e-06,
+      "loss": 0.5781806707382202,
+      "memory(GiB)": 128.42,
+      "step": 25,
+      "token_acc": 0.8080684596577017,
+      "train_speed(iter/s)": 0.213071
+    },
+    {
+      "epoch": 0.11607142857142858,
+      "grad_norm": 8.002751969235684,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 0.572704553604126,
+      "memory(GiB)": 128.42,
+      "step": 26,
+      "token_acc": 0.7769130998702983,
+      "train_speed(iter/s)": 0.213503
+    },
+    {
+      "epoch": 0.12053571428571429,
+      "grad_norm": 4.262323152935433,
+      "learning_rate": 6e-06,
+      "loss": 0.5535041689872742,
+      "memory(GiB)": 128.42,
+      "step": 27,
+      "token_acc": 0.8503253796095445,
+      "train_speed(iter/s)": 0.214104
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 3.966702275668715,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 0.6064645051956177,
+      "memory(GiB)": 128.42,
+      "step": 28,
+      "token_acc": 0.8506348020911128,
+      "train_speed(iter/s)": 0.214658
+    },
+    {
+      "epoch": 0.12946428571428573,
+      "grad_norm": 3.8567123289596172,
+      "learning_rate": 6.444444444444445e-06,
+      "loss": 0.5491228103637695,
+      "memory(GiB)": 128.42,
+      "step": 29,
+      "token_acc": 0.8545454545454545,
+      "train_speed(iter/s)": 0.215116
+    },
+    {
+      "epoch": 0.13392857142857142,
+      "grad_norm": 3.710395065072247,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.5767688751220703,
+      "memory(GiB)": 128.42,
+      "step": 30,
+      "token_acc": 0.84375,
+      "train_speed(iter/s)": 0.215602
+    },
+    {
+      "epoch": 0.13839285714285715,
+      "grad_norm": 4.281979952710429,
+      "learning_rate": 6.88888888888889e-06,
+      "loss": 0.5191075205802917,
+      "memory(GiB)": 128.42,
+      "step": 31,
+      "token_acc": 0.8571428571428571,
+      "train_speed(iter/s)": 0.216142
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 3.7571532580897102,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 0.5514931082725525,
+      "memory(GiB)": 128.42,
+      "step": 32,
+      "token_acc": 0.8145224940805051,
+      "train_speed(iter/s)": 0.216598
+    },
+    {
+      "epoch": 0.14732142857142858,
+      "grad_norm": 3.597514204800206,
+      "learning_rate": 7.333333333333333e-06,
+      "loss": 0.5634205341339111,
+      "memory(GiB)": 128.42,
+      "step": 33,
+      "token_acc": 0.8357963875205254,
+      "train_speed(iter/s)": 0.217011
+    },
+    {
+      "epoch": 0.15178571428571427,
+      "grad_norm": 3.511624738132458,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 0.5661548376083374,
+      "memory(GiB)": 128.42,
+      "step": 34,
+      "token_acc": 0.8533635676492819,
+      "train_speed(iter/s)": 0.217375
+    },
+    {
+      "epoch": 0.15625,
+      "grad_norm": 3.5093325994826454,
+      "learning_rate": 7.77777777777778e-06,
+      "loss": 0.5768500566482544,
+      "memory(GiB)": 128.42,
+      "step": 35,
+      "token_acc": 0.8479952124476361,
+      "train_speed(iter/s)": 0.217665
+    },
+    {
+      "epoch": 0.16071428571428573,
+      "grad_norm": 3.7955307572836268,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.5004974007606506,
+      "memory(GiB)": 128.42,
+      "step": 36,
+      "token_acc": 0.8147471054235222,
+      "train_speed(iter/s)": 0.217992
+    },
+    {
+      "epoch": 0.16517857142857142,
+      "grad_norm": 3.4615768583761954,
+      "learning_rate": 8.222222222222222e-06,
+      "loss": 0.5323991775512695,
+      "memory(GiB)": 128.42,
+      "step": 37,
+      "token_acc": 0.8320715760495526,
+      "train_speed(iter/s)": 0.218325
+    },
+    {
+      "epoch": 0.16964285714285715,
+      "grad_norm": 3.3517149428315505,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 0.5705447196960449,
+      "memory(GiB)": 128.42,
+      "step": 38,
+      "token_acc": 0.8195197521301317,
+      "train_speed(iter/s)": 0.21859
+    },
+    {
+      "epoch": 0.17410714285714285,
+      "grad_norm": 3.579209602352775,
+      "learning_rate": 8.666666666666668e-06,
+      "loss": 0.5391894578933716,
+      "memory(GiB)": 128.42,
+      "step": 39,
+      "token_acc": 0.8760736196319019,
+      "train_speed(iter/s)": 0.218783
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 3.4898869370386243,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 0.49445533752441406,
+      "memory(GiB)": 128.42,
+      "step": 40,
+      "token_acc": 0.883495145631068,
+      "train_speed(iter/s)": 0.21908
+    },
+    {
+      "epoch": 0.18303571428571427,
+      "grad_norm": 8.989941754990463,
+      "learning_rate": 9.111111111111112e-06,
+      "loss": 0.49502032995224,
+      "memory(GiB)": 128.42,
+      "step": 41,
+      "token_acc": 0.8318452380952381,
+      "train_speed(iter/s)": 0.219326
+    },
+    {
+      "epoch": 0.1875,
+      "grad_norm": 3.592536913293317,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.5180964469909668,
+      "memory(GiB)": 128.42,
+      "step": 42,
+      "token_acc": 0.8248995983935743,
+      "train_speed(iter/s)": 0.21958
+    },
+    {
+      "epoch": 0.19196428571428573,
+      "grad_norm": 3.3595948687960897,
+      "learning_rate": 9.555555555555556e-06,
+      "loss": 0.48923468589782715,
+      "memory(GiB)": 128.42,
+      "step": 43,
+      "token_acc": 0.8549222797927462,
+      "train_speed(iter/s)": 0.219825
+    },
+    {
+      "epoch": 0.19642857142857142,
+      "grad_norm": 3.254464883159724,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 0.4752197861671448,
+      "memory(GiB)": 128.42,
+      "step": 44,
+      "token_acc": 0.8691341705221415,
+      "train_speed(iter/s)": 0.220006
+    },
+    {
+      "epoch": 0.20089285714285715,
+      "grad_norm": 3.4895034998885666,
+      "learning_rate": 1e-05,
+      "loss": 0.46786195039749146,
+      "memory(GiB)": 128.42,
+      "step": 45,
+      "token_acc": 0.8913894324853229,
+      "train_speed(iter/s)": 0.220218
+    },
+    {
+      "epoch": 0.20535714285714285,
+      "grad_norm": 3.536205201848946,
+      "learning_rate": 9.999965929371846e-06,
+      "loss": 0.490744411945343,
+      "memory(GiB)": 128.42,
+      "step": 46,
+      "token_acc": 0.889745566692367,
+      "train_speed(iter/s)": 0.220409
+    },
+    {
+      "epoch": 0.20982142857142858,
+      "grad_norm": 3.4060931659065465,
+      "learning_rate": 9.999863717951704e-06,
+      "loss": 0.4735819101333618,
+      "memory(GiB)": 128.42,
+      "step": 47,
+      "token_acc": 0.839968152866242,
+      "train_speed(iter/s)": 0.220607
+    },
+    {
+      "epoch": 0.21428571428571427,
+      "grad_norm": 3.5259252637192753,
+      "learning_rate": 9.999693367132534e-06,
+      "loss": 0.4596032202243805,
+      "memory(GiB)": 128.42,
+      "step": 48,
+      "token_acc": 0.8322520852641334,
+      "train_speed(iter/s)": 0.220805
+    },
+    {
+      "epoch": 0.21875,
+      "grad_norm": 3.342810881695556,
+      "learning_rate": 9.999454879235926e-06,
+      "loss": 0.4986742436885834,
+      "memory(GiB)": 128.42,
+      "step": 49,
+      "token_acc": 0.8661538461538462,
+      "train_speed(iter/s)": 0.221017
+    },
+    {
+      "epoch": 0.22321428571428573,
+      "grad_norm": 3.4094938255821097,
+      "learning_rate": 9.99914825751205e-06,
+      "loss": 0.45326530933380127,
+      "memory(GiB)": 128.42,
+      "step": 50,
+      "token_acc": 0.8617771509167842,
+      "train_speed(iter/s)": 0.221194
+    },
+    {
+      "epoch": 0.22767857142857142,
+      "grad_norm": 3.366606187311854,
+      "learning_rate": 9.998773506139622e-06,
+      "loss": 0.49953100085258484,
+      "memory(GiB)": 128.42,
+      "step": 51,
+      "token_acc": 0.8632010943912448,
+      "train_speed(iter/s)": 0.221244
+    },
+    {
+      "epoch": 0.23214285714285715,
+      "grad_norm": 3.342945040893082,
+      "learning_rate": 9.998330630225853e-06,
+      "loss": 0.4754818081855774,
+      "memory(GiB)": 128.42,
+      "step": 52,
+      "token_acc": 0.8538116591928251,
+      "train_speed(iter/s)": 0.221426
+    },
+    {
+      "epoch": 0.23660714285714285,
+      "grad_norm": 3.2984275728119874,
+      "learning_rate": 9.99781963580636e-06,
+      "loss": 0.4748860001564026,
+      "memory(GiB)": 128.42,
+      "step": 53,
+      "token_acc": 0.8231116849580374,
+      "train_speed(iter/s)": 0.221635
+    },
+    {
+      "epoch": 0.24107142857142858,
+      "grad_norm": 3.1307130443222406,
+      "learning_rate": 9.997240529845112e-06,
+      "loss": 0.42877909541130066,
+      "memory(GiB)": 128.42,
+      "step": 54,
+      "token_acc": 0.8777173913043478,
+      "train_speed(iter/s)": 0.221813
+    },
+    {
+      "epoch": 0.24553571428571427,
+      "grad_norm": 3.1327655583720126,
+      "learning_rate": 9.996593320234302e-06,
+      "loss": 0.4814930856227875,
+      "memory(GiB)": 128.42,
+      "step": 55,
+      "token_acc": 0.865686274509804,
+      "train_speed(iter/s)": 0.221971
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 3.0672140778239676,
+      "learning_rate": 9.995878015794271e-06,
+      "loss": 0.4632089138031006,
+      "memory(GiB)": 128.42,
+      "step": 56,
+      "token_acc": 0.8444940476190477,
+      "train_speed(iter/s)": 0.222117
+    },
+    {
+      "epoch": 0.2544642857142857,
+      "grad_norm": 3.4023664393185724,
+      "learning_rate": 9.995094626273365e-06,
+      "loss": 0.501305103302002,
+      "memory(GiB)": 128.42,
+      "step": 57,
+      "token_acc": 0.8551660516605166,
+      "train_speed(iter/s)": 0.222231
+    },
+    {
+      "epoch": 0.25892857142857145,
+      "grad_norm": 2.908553060683868,
+      "learning_rate": 9.994243162347814e-06,
+      "loss": 0.41847342252731323,
+      "memory(GiB)": 128.42,
+      "step": 58,
+      "token_acc": 0.8760233918128655,
+      "train_speed(iter/s)": 0.222366
+    },
+    {
+      "epoch": 0.26339285714285715,
+      "grad_norm": 2.808351908407882,
+      "learning_rate": 9.993323635621582e-06,
+      "loss": 0.47732818126678467,
+      "memory(GiB)": 128.42,
+      "step": 59,
+      "token_acc": 0.8490945674044266,
+      "train_speed(iter/s)": 0.222388
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 2.7565786085479274,
+      "learning_rate": 9.992336058626212e-06,
+      "loss": 0.4374464452266693,
+      "memory(GiB)": 128.42,
+      "step": 60,
+      "token_acc": 0.8333333333333334,
+      "train_speed(iter/s)": 0.222532
+    },
+    {
+      "epoch": 0.27232142857142855,
+      "grad_norm": 2.500488866173154,
+      "learning_rate": 9.991280444820647e-06,
+      "loss": 0.4108882546424866,
+      "memory(GiB)": 128.42,
+      "step": 61,
+      "token_acc": 0.8650176678445229,
+      "train_speed(iter/s)": 0.222682
+    },
+    {
+      "epoch": 0.2767857142857143,
+      "grad_norm": 2.3349742419431676,
+      "learning_rate": 9.990156808591064e-06,
+      "loss": 0.43476420640945435,
+      "memory(GiB)": 128.42,
+      "step": 62,
+      "token_acc": 0.8691099476439791,
+      "train_speed(iter/s)": 0.222807
+    },
+    {
+      "epoch": 0.28125,
+      "grad_norm": 2.2421342964515407,
+      "learning_rate": 9.988965165250654e-06,
+      "loss": 0.4363088011741638,
+      "memory(GiB)": 128.42,
+      "step": 63,
+      "token_acc": 0.8770161290322581,
+      "train_speed(iter/s)": 0.222953
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 2.314693697676626,
+      "learning_rate": 9.987705531039434e-06,
+      "loss": 0.48462244868278503,
+      "memory(GiB)": 128.42,
+      "step": 64,
+      "token_acc": 0.8437990580847724,
+      "train_speed(iter/s)": 0.223033
+    },
+    {
+      "epoch": 0.29017857142857145,
+      "grad_norm": 2.404628201722386,
+      "learning_rate": 9.986377923124016e-06,
+      "loss": 0.46896690130233765,
+      "memory(GiB)": 128.42,
+      "step": 65,
+      "token_acc": 0.8426601784266018,
+      "train_speed(iter/s)": 0.223118
+    },
+    {
+      "epoch": 0.29464285714285715,
+      "grad_norm": 2.4623641287897224,
+      "learning_rate": 9.984982359597373e-06,
+      "loss": 0.43249690532684326,
+      "memory(GiB)": 128.42,
+      "step": 66,
+      "token_acc": 0.8550592525068368,
+      "train_speed(iter/s)": 0.223248
+    },
+    {
+      "epoch": 0.29910714285714285,
+      "grad_norm": 2.227239059437109,
+      "learning_rate": 9.983518859478597e-06,
+      "loss": 0.42663565278053284,
+      "memory(GiB)": 128.42,
+      "step": 67,
+      "token_acc": 0.8267526188557615,
+      "train_speed(iter/s)": 0.223364
+    },
+    {
+      "epoch": 0.30357142857142855,
+      "grad_norm": 2.152719299561224,
+      "learning_rate": 9.981987442712634e-06,
+      "loss": 0.42308685183525085,
+      "memory(GiB)": 128.42,
+      "step": 68,
+      "token_acc": 0.8523769808173478,
+      "train_speed(iter/s)": 0.223446
+    },
+    {
+      "epoch": 0.3080357142857143,
+      "grad_norm": 2.128383011723548,
+      "learning_rate": 9.980388130170015e-06,
+      "loss": 0.4897618293762207,
+      "memory(GiB)": 128.42,
+      "step": 69,
+      "token_acc": 0.8330404217926186,
+      "train_speed(iter/s)": 0.223543
+    },
+    {
+      "epoch": 0.3125,
+      "grad_norm": 2.058851793564121,
+      "learning_rate": 9.978720943646578e-06,
+      "loss": 0.4273832440376282,
+      "memory(GiB)": 128.42,
+      "step": 70,
+      "token_acc": 0.8670212765957447,
+      "train_speed(iter/s)": 0.223628
+    },
+    {
+      "epoch": 0.3169642857142857,
+      "grad_norm": 2.152936819995939,
+      "learning_rate": 9.976985905863155e-06,
+      "loss": 0.4190593361854553,
+      "memory(GiB)": 128.42,
+      "step": 71,
+      "token_acc": 0.8797145769622834,
+      "train_speed(iter/s)": 0.223737
+    },
+    {
+      "epoch": 0.32142857142857145,
+      "grad_norm": 2.021590225323182,
+      "learning_rate": 9.975183040465279e-06,
+      "loss": 0.3998218774795532,
+      "memory(GiB)": 128.42,
+      "step": 72,
+      "token_acc": 0.8594507269789984,
+      "train_speed(iter/s)": 0.223782
+    },
+    {
+      "epoch": 0.32589285714285715,
+      "grad_norm": 2.3074914591810103,
+      "learning_rate": 9.973312372022853e-06,
+      "loss": 0.44243016839027405,
+      "memory(GiB)": 128.42,
+      "step": 73,
+      "token_acc": 0.8616207951070336,
+      "train_speed(iter/s)": 0.223836
+    },
+    {
+      "epoch": 0.33035714285714285,
+      "grad_norm": 2.334012104081163,
+      "learning_rate": 9.971373926029815e-06,
+      "loss": 0.3950923681259155,
+      "memory(GiB)": 128.42,
+      "step": 74,
+      "token_acc": 0.868,
+      "train_speed(iter/s)": 0.22386
+    },
+    {
+      "epoch": 0.33482142857142855,
+      "grad_norm": 2.122646560469079,
+      "learning_rate": 9.969367728903796e-06,
+      "loss": 0.4495946168899536,
+      "memory(GiB)": 128.42,
+      "step": 75,
+      "token_acc": 0.8410689170182841,
+      "train_speed(iter/s)": 0.223964
+    },
+    {
+      "epoch": 0.3392857142857143,
+      "grad_norm": 1.9740961761354934,
+      "learning_rate": 9.967293807985754e-06,
+      "loss": 0.43762198090553284,
+      "memory(GiB)": 128.42,
+      "step": 76,
+      "token_acc": 0.8256802721088435,
+      "train_speed(iter/s)": 0.224053
+    },
+    {
+      "epoch": 0.34375,
+      "grad_norm": 2.142762356279292,
+      "learning_rate": 9.965152191539602e-06,
+      "loss": 0.4208489656448364,
+      "memory(GiB)": 128.42,
+      "step": 77,
+      "token_acc": 0.8441158348736907,
+      "train_speed(iter/s)": 0.224065
+    },
+    {
+      "epoch": 0.3482142857142857,
+      "grad_norm": 2.170173305950217,
+      "learning_rate": 9.96294290875183e-06,
+      "loss": 0.44035041332244873,
+      "memory(GiB)": 128.42,
+      "step": 78,
+      "token_acc": 0.8557422969187675,
+      "train_speed(iter/s)": 0.224153
+    },
+    {
+      "epoch": 0.35267857142857145,
+      "grad_norm": 2.0939957380279877,
+      "learning_rate": 9.960665989731099e-06,
+      "loss": 0.455330491065979,
+      "memory(GiB)": 128.42,
+      "step": 79,
+      "token_acc": 0.8592162554426706,
+      "train_speed(iter/s)": 0.224244
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 1.8572342974477305,
+      "learning_rate": 9.95832146550783e-06,
+      "loss": 0.42493194341659546,
+      "memory(GiB)": 128.42,
+      "step": 80,
+      "token_acc": 0.8617121354656632,
+      "train_speed(iter/s)": 0.224289
+    },
+    {
+      "epoch": 0.36160714285714285,
+      "grad_norm": 2.0697549040221017,
+      "learning_rate": 9.955909368033793e-06,
+      "loss": 0.38421130180358887,
+      "memory(GiB)": 128.42,
+      "step": 81,
+      "token_acc": 0.8648,
+      "train_speed(iter/s)": 0.224369
+    },
+    {
+      "epoch": 0.36607142857142855,
+      "grad_norm": 2.219370159090403,
+      "learning_rate": 9.953429730181653e-06,
+      "loss": 0.42679229378700256,
+      "memory(GiB)": 128.42,
+      "step": 82,
+      "token_acc": 0.8590038314176245,
+      "train_speed(iter/s)": 0.224448
+    },
+    {
+      "epoch": 0.3705357142857143,
+      "grad_norm": 2.0309694729211976,
+      "learning_rate": 9.950882585744544e-06,
+      "loss": 0.42348772287368774,
+      "memory(GiB)": 128.42,
+      "step": 83,
+      "token_acc": 0.8685220729366603,
+      "train_speed(iter/s)": 0.224528
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 1.9953416437787699,
+      "learning_rate": 9.948267969435585e-06,
+      "loss": 0.4190869927406311,
+      "memory(GiB)": 128.42,
+      "step": 84,
+      "token_acc": 0.8526682134570766,
+      "train_speed(iter/s)": 0.22461
+    },
+    {
+      "epoch": 0.3794642857142857,
+      "grad_norm": 2.066239379356436,
+      "learning_rate": 9.945585916887428e-06,
+      "loss": 0.4057341516017914,
+      "memory(GiB)": 128.42,
+      "step": 85,
+      "token_acc": 0.8572555205047319,
+      "train_speed(iter/s)": 0.224673
+    },
+    {
+      "epoch": 0.38392857142857145,
+      "grad_norm": 2.1617655431273186,
+      "learning_rate": 9.942836464651755e-06,
+      "loss": 0.4227410554885864,
+      "memory(GiB)": 128.42,
+      "step": 86,
+      "token_acc": 0.8300704776820673,
+      "train_speed(iter/s)": 0.224755
+    },
+    {
+      "epoch": 0.38839285714285715,
+      "grad_norm": 2.021096735054414,
+      "learning_rate": 9.940019650198796e-06,
+      "loss": 0.4023309051990509,
+      "memory(GiB)": 128.42,
+      "step": 87,
+      "token_acc": 0.8765765765765766,
+      "train_speed(iter/s)": 0.224785
+    },
+    {
+      "epoch": 0.39285714285714285,
+      "grad_norm": 1.84828349187564,
+      "learning_rate": 9.937135511916802e-06,
+      "loss": 0.4349609613418579,
+      "memory(GiB)": 128.42,
+      "step": 88,
+      "token_acc": 0.8710155670867309,
+      "train_speed(iter/s)": 0.224842
+    },
+    {
+      "epoch": 0.39732142857142855,
+      "grad_norm": 1.8764163466409773,
+      "learning_rate": 9.934184089111539e-06,
+      "loss": 0.4185750186443329,
+      "memory(GiB)": 128.42,
+      "step": 89,
+      "token_acc": 0.8533333333333334,
+      "train_speed(iter/s)": 0.224865
+    },
+    {
+      "epoch": 0.4017857142857143,
+      "grad_norm": 1.8632398207712257,
+      "learning_rate": 9.931165422005734e-06,
+      "loss": 0.438082754611969,
+      "memory(GiB)": 128.42,
+      "step": 90,
+      "token_acc": 0.8347417840375587,
+      "train_speed(iter/s)": 0.224921
+    },
+    {
+      "epoch": 0.40625,
+      "grad_norm": 2.2173701934991477,
+      "learning_rate": 9.928079551738542e-06,
+      "loss": 0.43483027815818787,
+      "memory(GiB)": 128.42,
+      "step": 91,
+      "token_acc": 0.8567454798331016,
+      "train_speed(iter/s)": 0.224959
+    },
+    {
+      "epoch": 0.4107142857142857,
+      "grad_norm": 2.016109177102097,
+      "learning_rate": 9.92492652036498e-06,
+      "loss": 0.4152158498764038,
+      "memory(GiB)": 128.42,
+      "step": 92,
+      "token_acc": 0.8462462462462462,
+      "train_speed(iter/s)": 0.225036
+    },
+    {
+      "epoch": 0.41517857142857145,
+      "grad_norm": 2.0334417717146978,
+      "learning_rate": 9.921706370855353e-06,
+      "loss": 0.3815462589263916,
+      "memory(GiB)": 128.42,
+      "step": 93,
+      "token_acc": 0.8731203007518797,
+      "train_speed(iter/s)": 0.225079
+    },
+    {
+      "epoch": 0.41964285714285715,
+      "grad_norm": 2.0732022690706198,
+      "learning_rate": 9.918419147094663e-06,
+      "loss": 0.40599995851516724,
+      "memory(GiB)": 128.42,
+      "step": 94,
+      "token_acc": 0.8433382137628112,
+      "train_speed(iter/s)": 0.225114
+    },
+    {
+      "epoch": 0.42410714285714285,
+      "grad_norm": 2.0919041720198797,
+      "learning_rate": 9.915064893882025e-06,
+      "loss": 0.4103803038597107,
+      "memory(GiB)": 128.42,
+      "step": 95,
+      "token_acc": 0.8777555110220441,
+      "train_speed(iter/s)": 0.225148
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 3.0894564290504496,
+      "learning_rate": 9.911643656930044e-06,
+      "loss": 0.441211998462677,
+      "memory(GiB)": 128.42,
+      "step": 96,
+      "token_acc": 0.8468013468013468,
+      "train_speed(iter/s)": 0.225197
+    },
+    {
+      "epoch": 0.4330357142857143,
+      "grad_norm": 1.9988119785462557,
+      "learning_rate": 9.908155482864194e-06,
+      "loss": 0.45225849747657776,
+      "memory(GiB)": 128.42,
+      "step": 97,
+      "token_acc": 0.8370786516853933,
+      "train_speed(iter/s)": 0.225271
+    },
+    {
+      "epoch": 0.4375,
+      "grad_norm": 2.342392393484222,
+      "learning_rate": 9.904600419222191e-06,
+      "loss": 0.38248753547668457,
+      "memory(GiB)": 128.42,
+      "step": 98,
+      "token_acc": 0.9082652134423251,
+      "train_speed(iter/s)": 0.225303
+    },
+    {
+      "epoch": 0.4419642857142857,
+      "grad_norm": 2.709700792154672,
+      "learning_rate": 9.900978514453335e-06,
+      "loss": 0.43812692165374756,
+      "memory(GiB)": 128.42,
+      "step": 99,
+      "token_acc": 0.8883374689826302,
+      "train_speed(iter/s)": 0.225353
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 2.450946542331287,
+      "learning_rate": 9.897289817917852e-06,
+      "loss": 0.41455474495887756,
+      "memory(GiB)": 128.42,
+      "step": 100,
+      "token_acc": 0.8553672316384181,
+      "train_speed(iter/s)": 0.225398
+    },
+    {
+      "epoch": 0.45089285714285715,
+      "grad_norm": 1.9224997517284426,
+      "learning_rate": 9.893534379886229e-06,
+      "loss": 0.3818276822566986,
+      "memory(GiB)": 128.42,
+      "step": 101,
+      "token_acc": 0.8751062022090059,
+      "train_speed(iter/s)": 0.225453
+    },
+    {
+      "epoch": 0.45535714285714285,
+      "grad_norm": 1.9760417729359396,
+      "learning_rate": 9.889712251538516e-06,
+      "loss": 0.4196687936782837,
+      "memory(GiB)": 128.42,
+      "step": 102,
+      "token_acc": 0.8859784283513097,
+      "train_speed(iter/s)": 0.225506
+    },
+    {
+      "epoch": 0.45982142857142855,
+      "grad_norm": 1.9615231678461282,
+      "learning_rate": 9.88582348496364e-06,
+      "loss": 0.41014599800109863,
+      "memory(GiB)": 128.42,
+      "step": 103,
+      "token_acc": 0.8596377749029754,
+      "train_speed(iter/s)": 0.225567
+    },
+    {
+      "epoch": 0.4642857142857143,
+      "grad_norm": 2.053532677771882,
+      "learning_rate": 9.881868133158687e-06,
+      "loss": 0.3833034634590149,
+      "memory(GiB)": 128.42,
+      "step": 104,
+      "token_acc": 0.8368932038834952,
+      "train_speed(iter/s)": 0.225632
+    },
+    {
+      "epoch": 0.46875,
+      "grad_norm": 2.1912755777585526,
+      "learning_rate": 9.877846250028188e-06,
+      "loss": 0.40928328037261963,
+      "memory(GiB)": 128.42,
+      "step": 105,
+      "token_acc": 0.8811881188118812,
+      "train_speed(iter/s)": 0.225663
+    },
+    {
+      "epoch": 0.4732142857142857,
+      "grad_norm": 1.9575903894574092,
+      "learning_rate": 9.873757890383374e-06,
+      "loss": 0.41512739658355713,
+      "memory(GiB)": 128.42,
+      "step": 106,
+      "token_acc": 0.8568298027757487,
+      "train_speed(iter/s)": 0.225715
+    },
+    {
+      "epoch": 0.47767857142857145,
+      "grad_norm": 2.2673539760779136,
+      "learning_rate": 9.869603109941442e-06,
+      "loss": 0.41499513387680054,
+      "memory(GiB)": 128.42,
+      "step": 107,
+      "token_acc": 0.8683220466516177,
+      "train_speed(iter/s)": 0.225753
+    },
+    {
+      "epoch": 0.48214285714285715,
+      "grad_norm": 5.359243918313353,
+      "learning_rate": 9.865381965324778e-06,
+      "loss": 0.40060681104660034,
+      "memory(GiB)": 128.42,
+      "step": 108,
+      "token_acc": 0.876412429378531,
+      "train_speed(iter/s)": 0.22579
+    },
+    {
+      "epoch": 0.48660714285714285,
+      "grad_norm": 2.1052724949805257,
+      "learning_rate": 9.861094514060206e-06,
+      "loss": 0.45214220881462097,
+      "memory(GiB)": 128.42,
+      "step": 109,
+      "token_acc": 0.8455079764903443,
+      "train_speed(iter/s)": 0.225835
+    },
+    {
+      "epoch": 0.49107142857142855,
+      "grad_norm": 3.125050220505369,
+      "learning_rate": 9.856740814578186e-06,
+      "loss": 0.3993504047393799,
+      "memory(GiB)": 128.42,
+      "step": 110,
+      "token_acc": 0.8351555929352397,
+      "train_speed(iter/s)": 0.22589
+    },
+    {
+      "epoch": 0.4955357142857143,
+      "grad_norm": 1.8525790808401106,
+      "learning_rate": 9.85232092621203e-06,
+      "loss": 0.39937469363212585,
+      "memory(GiB)": 128.42,
+      "step": 111,
+      "token_acc": 0.8457446808510638,
+      "train_speed(iter/s)": 0.225948
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.9089549875623408,
+      "learning_rate": 9.847834909197089e-06,
+      "loss": 0.40335044264793396,
+      "memory(GiB)": 128.42,
+      "step": 112,
+      "token_acc": 0.8628019323671497,
+      "train_speed(iter/s)": 0.226007
+    },
+    {
+      "epoch": 0.5044642857142857,
+      "grad_norm": 1.9749493510077332,
+      "learning_rate": 9.843282824669926e-06,
+      "loss": 0.42468175292015076,
+      "memory(GiB)": 128.42,
+      "step": 113,
+      "token_acc": 0.8340108401084011,
+      "train_speed(iter/s)": 0.226034
+    },
+    {
+      "epoch": 0.5089285714285714,
+      "grad_norm": 1.9937044644554462,
+      "learning_rate": 9.838664734667496e-06,
+      "loss": 0.38329237699508667,
+      "memory(GiB)": 128.42,
+      "step": 114,
+      "token_acc": 0.8425443169968717,
+      "train_speed(iter/s)": 0.226058
+    },
+    {
+      "epoch": 0.5133928571428571,
+      "grad_norm": 1.560721136274801,
+      "learning_rate": 9.833980702126287e-06,
+      "loss": 0.3472307324409485,
+      "memory(GiB)": 128.42,
+      "step": 115,
+      "token_acc": 0.8923444976076556,
+      "train_speed(iter/s)": 0.226091
+    },
+    {
+      "epoch": 0.5178571428571429,
+      "grad_norm": 1.7863431958110831,
+      "learning_rate": 9.829230790881474e-06,
+      "loss": 0.4139295518398285,
+      "memory(GiB)": 128.42,
+      "step": 116,
+      "token_acc": 0.8657142857142858,
+      "train_speed(iter/s)": 0.226136
+    },
+    {
+      "epoch": 0.5223214285714286,
+      "grad_norm": 2.0227555853718804,
+      "learning_rate": 9.82441506566604e-06,
+      "loss": 0.4211953282356262,
+      "memory(GiB)": 128.42,
+      "step": 117,
+      "token_acc": 0.8500986193293886,
+      "train_speed(iter/s)": 0.226158
+    },
+    {
+      "epoch": 0.5267857142857143,
+      "grad_norm": 1.892221973169317,
+      "learning_rate": 9.819533592109898e-06,
+      "loss": 0.3711167573928833,
+      "memory(GiB)": 128.42,
+      "step": 118,
+      "token_acc": 0.8703849175176748,
+      "train_speed(iter/s)": 0.226201
+    },
+    {
+      "epoch": 0.53125,
+      "grad_norm": 1.8769689856557228,
+      "learning_rate": 9.814586436738998e-06,
+      "loss": 0.3753877282142639,
+      "memory(GiB)": 128.42,
+      "step": 119,
+      "token_acc": 0.8629655657062544,
+      "train_speed(iter/s)": 0.226223
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 1.9077137109160918,
+      "learning_rate": 9.809573666974413e-06,
+      "loss": 0.3845405876636505,
+      "memory(GiB)": 128.42,
+      "step": 120,
+      "token_acc": 0.8865248226950354,
+      "train_speed(iter/s)": 0.226266
+    },
+    {
+      "epoch": 0.5401785714285714,
+      "grad_norm": 1.8483195634970053,
+      "learning_rate": 9.80449535113143e-06,
+      "loss": 0.3865225315093994,
+      "memory(GiB)": 128.42,
+      "step": 121,
+      "token_acc": 0.8959227467811158,
+      "train_speed(iter/s)": 0.226304
+    },
+    {
+      "epoch": 0.5446428571428571,
+      "grad_norm": 2.015815422114459,
+      "learning_rate": 9.799351558418616e-06,
+      "loss": 0.4358481764793396,
+      "memory(GiB)": 128.42,
+      "step": 122,
+      "token_acc": 0.8597063621533442,
+      "train_speed(iter/s)": 0.226338
+    },
+    {
+      "epoch": 0.5491071428571429,
+      "grad_norm": 1.8365267081758667,
+      "learning_rate": 9.794142358936869e-06,
+      "loss": 0.4205068051815033,
+      "memory(GiB)": 128.42,
+      "step": 123,
+      "token_acc": 0.8647194465795542,
+      "train_speed(iter/s)": 0.226383
+    },
+    {
+      "epoch": 0.5535714285714286,
+      "grad_norm": 2.0725669814164362,
+      "learning_rate": 9.788867823678465e-06,
+      "loss": 0.42481255531311035,
+      "memory(GiB)": 128.42,
+      "step": 124,
+      "token_acc": 0.8483412322274881,
+      "train_speed(iter/s)": 0.226421
+    },
+    {
+      "epoch": 0.5580357142857143,
+      "grad_norm": 1.914935421654447,
+      "learning_rate": 9.7835280245261e-06,
+      "loss": 0.39662468433380127,
+      "memory(GiB)": 128.42,
+      "step": 125,
+      "token_acc": 0.8517520215633423,
+      "train_speed(iter/s)": 0.226451
+    },
+    {
+      "epoch": 0.5625,
+      "grad_norm": 1.8558613972881188,
+      "learning_rate": 9.778123034251898e-06,
+      "loss": 0.40216946601867676,
+      "memory(GiB)": 128.42,
+      "step": 126,
+      "token_acc": 0.8529411764705882,
+      "train_speed(iter/s)": 0.226489
+    },
+    {
+      "epoch": 0.5669642857142857,
+      "grad_norm": 2.073519670141887,
+      "learning_rate": 9.772652926516424e-06,
+      "loss": 0.4289711117744446,
+      "memory(GiB)": 128.42,
+      "step": 127,
+      "token_acc": 0.8672032193158954,
+      "train_speed(iter/s)": 0.226539
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 1.9393590937147855,
+      "learning_rate": 9.76711777586768e-06,
+      "loss": 0.411954402923584,
+      "memory(GiB)": 128.42,
+      "step": 128,
+      "token_acc": 0.8615771812080537,
+      "train_speed(iter/s)": 0.226549
+    },
+    {
+      "epoch": 0.5758928571428571,
+      "grad_norm": 1.700134216785422,
+      "learning_rate": 9.76151765774009e-06,
+      "loss": 0.38732337951660156,
+      "memory(GiB)": 128.42,
+      "step": 129,
+      "token_acc": 0.8750952018278751,
+      "train_speed(iter/s)": 0.226588
+    },
+    {
+      "epoch": 0.5803571428571429,
+      "grad_norm": 1.8772660862292765,
+      "learning_rate": 9.755852648453472e-06,
+      "loss": 0.4092137813568115,
+      "memory(GiB)": 128.42,
+      "step": 130,
+      "token_acc": 0.8649969456322542,
+      "train_speed(iter/s)": 0.226612
+    },
+    {
+      "epoch": 0.5848214285714286,
+      "grad_norm": 1.84397328756411,
+      "learning_rate": 9.750122825211995e-06,
+      "loss": 0.3621349334716797,
+      "memory(GiB)": 128.42,
+      "step": 131,
+      "token_acc": 0.9,
+      "train_speed(iter/s)": 0.226653
+    },
+    {
+      "epoch": 0.5892857142857143,
+      "grad_norm": 1.8410084418847592,
+      "learning_rate": 9.74432826610313e-06,
+      "loss": 0.3831413984298706,
+      "memory(GiB)": 128.42,
+      "step": 132,
+      "token_acc": 0.8408723747980614,
+      "train_speed(iter/s)": 0.226689
+    },
+    {
+      "epoch": 0.59375,
+      "grad_norm": 1.8350848291819015,
+      "learning_rate": 9.738469050096583e-06,
+      "loss": 0.4100695252418518,
+      "memory(GiB)": 128.42,
+      "step": 133,
+      "token_acc": 0.8534258456201215,
+      "train_speed(iter/s)": 0.226704
+    },
+    {
+      "epoch": 0.5982142857142857,
+      "grad_norm": 1.8565849327061523,
+      "learning_rate": 9.732545257043224e-06,
+      "loss": 0.42670905590057373,
+      "memory(GiB)": 128.42,
+      "step": 134,
+      "token_acc": 0.8765670202507232,
+      "train_speed(iter/s)": 0.226619
+    },
+    {
+      "epoch": 0.6026785714285714,
+      "grad_norm": 1.8345425226029803,
+      "learning_rate": 9.726556967673992e-06,
+      "loss": 0.4107423424720764,
+      "memory(GiB)": 128.42,
+      "step": 135,
+      "token_acc": 0.8629764065335753,
+      "train_speed(iter/s)": 0.226641
+    },
+    {
+      "epoch": 0.6071428571428571,
+      "grad_norm": 1.8197069879254395,
+      "learning_rate": 9.720504263598802e-06,
+      "loss": 0.3949946463108063,
+      "memory(GiB)": 128.42,
+      "step": 136,
+      "token_acc": 0.8668706962509564,
+      "train_speed(iter/s)": 0.226655
+    },
+    {
+      "epoch": 0.6116071428571429,
+      "grad_norm": 1.8605070813229492,
+      "learning_rate": 9.714387227305422e-06,
+      "loss": 0.3933284878730774,
+      "memory(GiB)": 128.42,
+      "step": 137,
+      "token_acc": 0.8596247394023627,
+      "train_speed(iter/s)": 0.226682
+    },
+    {
+      "epoch": 0.6160714285714286,
+      "grad_norm": 1.9607719811040827,
+      "learning_rate": 9.708205942158362e-06,
+      "loss": 0.37001386284828186,
+      "memory(GiB)": 128.42,
+      "step": 138,
+      "token_acc": 0.8576923076923076,
+      "train_speed(iter/s)": 0.226702
+    },
+    {
+      "epoch": 0.6205357142857143,
+      "grad_norm": 1.7714294063492095,
+      "learning_rate": 9.701960492397727e-06,
+      "loss": 0.40147697925567627,
+      "memory(GiB)": 128.42,
+      "step": 139,
+      "token_acc": 0.8800834202294057,
+      "train_speed(iter/s)": 0.226715
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 1.9248044276022995,
+      "learning_rate": 9.695650963138079e-06,
+      "loss": 0.4316914677619934,
+      "memory(GiB)": 128.42,
+      "step": 140,
+      "token_acc": 0.8928994082840237,
+      "train_speed(iter/s)": 0.226745
+    },
+    {
+      "epoch": 0.6294642857142857,
+      "grad_norm": 1.9422362404479026,
+      "learning_rate": 9.689277440367263e-06,
+      "loss": 0.43073832988739014,
+      "memory(GiB)": 128.42,
+      "step": 141,
+      "token_acc": 0.8460329929300864,
+      "train_speed(iter/s)": 0.226779
+    },
+    {
+      "epoch": 0.6339285714285714,
+      "grad_norm": 1.8593524333480311,
+      "learning_rate": 9.682840010945254e-06,
+      "loss": 0.392534077167511,
+      "memory(GiB)": 128.42,
+      "step": 142,
+      "token_acc": 0.9042076991942704,
+      "train_speed(iter/s)": 0.226798
+    },
+    {
+      "epoch": 0.6383928571428571,
+      "grad_norm": 1.924142438232916,
+      "learning_rate": 9.676338762602954e-06,
+      "loss": 0.3955026865005493,
+      "memory(GiB)": 128.42,
+      "step": 143,
+      "token_acc": 0.8868398994132439,
+      "train_speed(iter/s)": 0.226803
+    },
+    {
+      "epoch": 0.6428571428571429,
+      "grad_norm": 1.803030550107758,
+      "learning_rate": 9.669773783941012e-06,
+      "loss": 0.38179874420166016,
+      "memory(GiB)": 128.42,
+      "step": 144,
+      "token_acc": 0.853448275862069,
+      "train_speed(iter/s)": 0.226841
+    },
+    {
+      "epoch": 0.6473214285714286,
+      "grad_norm": 2.136428096666585,
+      "learning_rate": 9.663145164428606e-06,
+      "loss": 0.43818652629852295,
+      "memory(GiB)": 128.42,
+      "step": 145,
+      "token_acc": 0.8600973236009732,
+      "train_speed(iter/s)": 0.226848
+    },
+    {
+      "epoch": 0.6517857142857143,
+      "grad_norm": 1.9680755115856965,
+      "learning_rate": 9.656452994402225e-06,
+      "loss": 0.38914287090301514,
+      "memory(GiB)": 128.42,
+      "step": 146,
+      "token_acc": 0.8546511627906976,
+      "train_speed(iter/s)": 0.226865
+    },
+    {
+      "epoch": 0.65625,
+      "grad_norm": 1.8644180163525486,
+      "learning_rate": 9.649697365064448e-06,
+      "loss": 0.3467503786087036,
+      "memory(GiB)": 128.42,
+      "step": 147,
+      "token_acc": 0.8939580764488286,
+      "train_speed(iter/s)": 0.226873
+    },
+    {
+      "epoch": 0.6607142857142857,
+      "grad_norm": 1.7646489123753146,
+      "learning_rate": 9.642878368482685e-06,
+      "loss": 0.40651845932006836,
+      "memory(GiB)": 128.42,
+      "step": 148,
+      "token_acc": 0.8590006844626967,
+      "train_speed(iter/s)": 0.226905
+    },
+    {
+      "epoch": 0.6651785714285714,
+      "grad_norm": 1.7110984383437826,
+      "learning_rate": 9.635996097587938e-06,
+      "loss": 0.3564274311065674,
+      "memory(GiB)": 128.42,
+      "step": 149,
+      "token_acc": 0.8814642649622313,
+      "train_speed(iter/s)": 0.226926
+    },
+    {
+      "epoch": 0.6696428571428571,
+      "grad_norm": 1.8656412445393,
+      "learning_rate": 9.629050646173521e-06,
+      "loss": 0.4157032370567322,
+      "memory(GiB)": 128.42,
+      "step": 150,
+      "token_acc": 0.8456883509833586,
+      "train_speed(iter/s)": 0.22696
+    },
+    {
+      "epoch": 0.6741071428571429,
+      "grad_norm": 1.7627569933191594,
+      "learning_rate": 9.622042108893794e-06,
+      "loss": 0.38666343688964844,
+      "memory(GiB)": 128.42,
+      "step": 151,
+      "token_acc": 0.8736532810969637,
+      "train_speed(iter/s)": 0.226987
+    },
+    {
+      "epoch": 0.6785714285714286,
+      "grad_norm": 1.8386703451357664,
+      "learning_rate": 9.614970581262863e-06,
+      "loss": 0.4122501611709595,
+      "memory(GiB)": 128.42,
+      "step": 152,
+      "token_acc": 0.8647302904564316,
+      "train_speed(iter/s)": 0.227012
+    },
+    {
+      "epoch": 0.6830357142857143,
+      "grad_norm": 1.7836526981478449,
+      "learning_rate": 9.607836159653283e-06,
+      "loss": 0.3835086226463318,
+      "memory(GiB)": 128.42,
+      "step": 153,
+      "token_acc": 0.8738277919863597,
+      "train_speed(iter/s)": 0.227049
+    },
+    {
+      "epoch": 0.6875,
+      "grad_norm": 1.821283805318599,
+      "learning_rate": 9.600638941294744e-06,
+      "loss": 0.38780418038368225,
+      "memory(GiB)": 128.42,
+      "step": 154,
+      "token_acc": 0.8396739130434783,
+      "train_speed(iter/s)": 0.227066
+    },
+    {
+      "epoch": 0.6919642857142857,
+      "grad_norm": 1.86521272428445,
+      "learning_rate": 9.593379024272748e-06,
+      "loss": 0.44408848881721497,
+      "memory(GiB)": 128.42,
+      "step": 155,
+      "token_acc": 0.810251798561151,
+      "train_speed(iter/s)": 0.227065
+    },
+    {
+      "epoch": 0.6964285714285714,
+      "grad_norm": 1.6172624083533582,
+      "learning_rate": 9.586056507527266e-06,
+      "loss": 0.38970640301704407,
+      "memory(GiB)": 128.42,
+      "step": 156,
+      "token_acc": 0.8582302568981922,
+      "train_speed(iter/s)": 0.22708
+    },
+    {
+      "epoch": 0.7008928571428571,
+      "grad_norm": 1.7015443365556533,
+      "learning_rate": 9.578671490851396e-06,
+      "loss": 0.37300947308540344,
+      "memory(GiB)": 128.42,
+      "step": 157,
+      "token_acc": 0.8557409224730128,
+      "train_speed(iter/s)": 0.227102
+    },
+    {
+      "epoch": 0.7053571428571429,
+      "grad_norm": 1.9126023576603783,
+      "learning_rate": 9.571224074890004e-06,
+      "loss": 0.40674978494644165,
+      "memory(GiB)": 128.42,
+      "step": 158,
+      "token_acc": 0.8636613523914238,
+      "train_speed(iter/s)": 0.227116
+    },
+    {
+      "epoch": 0.7098214285714286,
+      "grad_norm": 1.8236785039444972,
+      "learning_rate": 9.563714361138343e-06,
+      "loss": 0.34416210651397705,
+      "memory(GiB)": 128.42,
+      "step": 159,
+      "token_acc": 0.8637236084452975,
+      "train_speed(iter/s)": 0.227134
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 1.737598943732038,
+      "learning_rate": 9.55614245194068e-06,
+      "loss": 0.40488338470458984,
+      "memory(GiB)": 128.42,
+      "step": 160,
+      "token_acc": 0.856110776640578,
+      "train_speed(iter/s)": 0.227158
+    },
+    {
+      "epoch": 0.71875,
+      "grad_norm": 1.6120450500571684,
+      "learning_rate": 9.548508450488895e-06,
+      "loss": 0.3595935106277466,
+      "memory(GiB)": 128.42,
+      "step": 161,
+      "token_acc": 0.8699132755170114,
+      "train_speed(iter/s)": 0.227195
+    },
+    {
+      "epoch": 0.7232142857142857,
+      "grad_norm": 1.693568310349621,
+      "learning_rate": 9.540812460821078e-06,
+      "loss": 0.352266401052475,
+      "memory(GiB)": 128.42,
+      "step": 162,
+      "token_acc": 0.8873072360616845,
+      "train_speed(iter/s)": 0.227208
+    },
+    {
+      "epoch": 0.7276785714285714,
+      "grad_norm": 1.5757683818467814,
+      "learning_rate": 9.533054587820112e-06,
+      "loss": 0.369427353143692,
+      "memory(GiB)": 128.42,
+      "step": 163,
+      "token_acc": 0.8559322033898306,
+      "train_speed(iter/s)": 0.227247
+    },
+    {
+      "epoch": 0.7321428571428571,
+      "grad_norm": 1.6076615751659218,
+      "learning_rate": 9.525234937212238e-06,
+      "loss": 0.3551783859729767,
+      "memory(GiB)": 128.42,
+      "step": 164,
+      "token_acc": 0.8765217391304347,
+      "train_speed(iter/s)": 0.227269
+    },
+    {
+      "epoch": 0.7366071428571429,
+      "grad_norm": 1.8347392969943477,
+      "learning_rate": 9.51735361556562e-06,
+      "loss": 0.38330310583114624,
+      "memory(GiB)": 128.42,
+      "step": 165,
+      "token_acc": 0.8470474346563408,
+      "train_speed(iter/s)": 0.227248
+    },
+    {
+      "epoch": 0.7410714285714286,
+      "grad_norm": 1.7630173128684303,
+      "learning_rate": 9.509410730288888e-06,
+      "loss": 0.37735095620155334,
+      "memory(GiB)": 128.42,
+      "step": 166,
+      "token_acc": 0.8636363636363636,
+      "train_speed(iter/s)": 0.227248
+    },
+    {
+      "epoch": 0.7455357142857143,
+      "grad_norm": 1.782005009352597,
+      "learning_rate": 9.501406389629682e-06,
+      "loss": 0.3654155135154724,
+      "memory(GiB)": 128.42,
+      "step": 167,
+      "token_acc": 0.8790035587188612,
+      "train_speed(iter/s)": 0.227271
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 1.7574512911157494,
+      "learning_rate": 9.493340702673162e-06,
+      "loss": 0.411441445350647,
+      "memory(GiB)": 128.42,
+      "step": 168,
+      "token_acc": 0.8481375358166189,
+      "train_speed(iter/s)": 0.227286
+    },
+    {
+      "epoch": 0.7544642857142857,
+      "grad_norm": 1.7576357797026396,
+      "learning_rate": 9.485213779340541e-06,
+      "loss": 0.4257476329803467,
+      "memory(GiB)": 128.42,
+      "step": 169,
+      "token_acc": 0.8625083836351441,
+      "train_speed(iter/s)": 0.22732
+    },
+    {
+      "epoch": 0.7589285714285714,
+      "grad_norm": 1.7527537543212042,
+      "learning_rate": 9.47702573038757e-06,
+      "loss": 0.39181047677993774,
+      "memory(GiB)": 128.42,
+      "step": 170,
+      "token_acc": 0.8573643410852713,
+      "train_speed(iter/s)": 0.227307
+    },
+    {
+      "epoch": 0.7633928571428571,
+      "grad_norm": 1.7629371359176924,
+      "learning_rate": 9.468776667403039e-06,
+      "loss": 0.4168310761451721,
+      "memory(GiB)": 128.42,
+      "step": 171,
+      "token_acc": 0.8672046955245781,
+      "train_speed(iter/s)": 0.227328
+    },
+    {
+      "epoch": 0.7678571428571429,
+      "grad_norm": 1.8049015584821484,
+      "learning_rate": 9.460466702807249e-06,
+      "loss": 0.4058079123497009,
+      "memory(GiB)": 128.42,
+      "step": 172,
+      "token_acc": 0.8550583657587548,
+      "train_speed(iter/s)": 0.227352
+    },
+    {
+      "epoch": 0.7723214285714286,
+      "grad_norm": 1.7714028210772665,
+      "learning_rate": 9.452095949850489e-06,
+      "loss": 0.3684278130531311,
+      "memory(GiB)": 128.42,
+      "step": 173,
+      "token_acc": 0.8781074578989575,
+      "train_speed(iter/s)": 0.22737
+    },
+    {
+      "epoch": 0.7767857142857143,
+      "grad_norm": 1.7353068281562303,
+      "learning_rate": 9.44366452261148e-06,
+      "loss": 0.37866702675819397,
+      "memory(GiB)": 128.42,
+      "step": 174,
+      "token_acc": 0.8828058169375534,
+      "train_speed(iter/s)": 0.227381
+    },
+    {
+      "epoch": 0.78125,
+      "grad_norm": 1.7300961263809764,
+      "learning_rate": 9.435172535995831e-06,
+      "loss": 0.38601797819137573,
+      "memory(GiB)": 128.42,
+      "step": 175,
+      "token_acc": 0.8662420382165605,
+      "train_speed(iter/s)": 0.227398
+    },
+    {
+      "epoch": 0.7857142857142857,
+      "grad_norm": 1.741272487827446,
+      "learning_rate": 9.426620105734473e-06,
+      "loss": 0.3633964955806732,
+      "memory(GiB)": 128.42,
+      "step": 176,
+      "token_acc": 0.9051339285714286,
+      "train_speed(iter/s)": 0.227429
+    },
+    {
+      "epoch": 0.7901785714285714,
+      "grad_norm": 1.6875950307747867,
+      "learning_rate": 9.41800734838207e-06,
+      "loss": 0.4060158431529999,
+      "memory(GiB)": 128.42,
+      "step": 177,
+      "token_acc": 0.8517985611510791,
+      "train_speed(iter/s)": 0.227457
+    },
+    {
+      "epoch": 0.7946428571428571,
+      "grad_norm": 1.6697656183460996,
+      "learning_rate": 9.409334381315445e-06,
+      "loss": 0.37892186641693115,
+      "memory(GiB)": 128.42,
+      "step": 178,
+      "token_acc": 0.876099706744868,
+      "train_speed(iter/s)": 0.227485
+    },
+    {
+      "epoch": 0.7991071428571429,
+      "grad_norm": 1.6519426865105165,
+      "learning_rate": 9.400601322731974e-06,
+      "loss": 0.3567098379135132,
+      "memory(GiB)": 128.42,
+      "step": 179,
+      "token_acc": 0.8501170960187353,
+      "train_speed(iter/s)": 0.227509
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 1.7786028639956994,
+      "learning_rate": 9.391808291647971e-06,
+      "loss": 0.37524616718292236,
+      "memory(GiB)": 128.42,
+      "step": 180,
+      "token_acc": 0.8761421319796955,
+      "train_speed(iter/s)": 0.227536
+    },
+    {
+      "epoch": 0.8080357142857143,
+      "grad_norm": 1.7489126178483658,
+      "learning_rate": 9.382955407897076e-06,
+      "loss": 0.41019266843795776,
+      "memory(GiB)": 128.42,
+      "step": 181,
+      "token_acc": 0.8840182648401826,
+      "train_speed(iter/s)": 0.227569
+    },
+    {
+      "epoch": 0.8125,
+      "grad_norm": 1.987286094728141,
+      "learning_rate": 9.37404279212861e-06,
+      "loss": 0.4129825830459595,
+      "memory(GiB)": 128.42,
+      "step": 182,
+      "token_acc": 0.8421052631578947,
+      "train_speed(iter/s)": 0.227601
+    },
+    {
+      "epoch": 0.8169642857142857,
+      "grad_norm": 1.9033194956801682,
+      "learning_rate": 9.365070565805941e-06,
+      "loss": 0.4232962131500244,
+      "memory(GiB)": 128.42,
+      "step": 183,
+      "token_acc": 0.8359788359788359,
+      "train_speed(iter/s)": 0.227614
+    },
+    {
+      "epoch": 0.8214285714285714,
+      "grad_norm": 1.8779587414450334,
+      "learning_rate": 9.356038851204826e-06,
+      "loss": 0.4073084592819214,
+      "memory(GiB)": 128.42,
+      "step": 184,
+      "token_acc": 0.8229548229548229,
+      "train_speed(iter/s)": 0.227615
+    },
+    {
+      "epoch": 0.8258928571428571,
+      "grad_norm": 1.9063659856430246,
+      "learning_rate": 9.346947771411738e-06,
+      "loss": 0.4089735746383667,
+      "memory(GiB)": 128.42,
+      "step": 185,
+      "token_acc": 0.8871391076115486,
+      "train_speed(iter/s)": 0.22762
+    },
+    {
+      "epoch": 0.8303571428571429,
+      "grad_norm": 1.4836242663153665,
+      "learning_rate": 9.337797450322197e-06,
+      "loss": 0.3512466549873352,
+      "memory(GiB)": 128.42,
+      "step": 186,
+      "token_acc": 0.872382851445663,
+      "train_speed(iter/s)": 0.227641
+    },
+    {
+      "epoch": 0.8348214285714286,
+      "grad_norm": 1.555500407882726,
+      "learning_rate": 9.328588012639081e-06,
+      "loss": 0.3591383099555969,
+      "memory(GiB)": 128.42,
+      "step": 187,
+      "token_acc": 0.8741721854304636,
+      "train_speed(iter/s)": 0.227662
+    },
+    {
+      "epoch": 0.8392857142857143,
+      "grad_norm": 1.7087345811385168,
+      "learning_rate": 9.319319583870918e-06,
+      "loss": 0.37500661611557007,
+      "memory(GiB)": 128.42,
+      "step": 188,
+      "token_acc": 0.8708685626441199,
+      "train_speed(iter/s)": 0.22768
+    },
+    {
+      "epoch": 0.84375,
+      "grad_norm": 1.7815220397240672,
+      "learning_rate": 9.309992290330182e-06,
+      "loss": 0.40438687801361084,
+      "memory(GiB)": 128.42,
+      "step": 189,
+      "token_acc": 0.8631436314363143,
+      "train_speed(iter/s)": 0.227689
+    },
+    {
+      "epoch": 0.8482142857142857,
+      "grad_norm": 1.5997044351272065,
+      "learning_rate": 9.300606259131578e-06,
+      "loss": 0.37673109769821167,
+      "memory(GiB)": 128.42,
+      "step": 190,
+      "token_acc": 0.8944281524926686,
+      "train_speed(iter/s)": 0.227709
+    },
+    {
+      "epoch": 0.8526785714285714,
+      "grad_norm": 1.7618126169744526,
+      "learning_rate": 9.291161618190295e-06,
+      "loss": 0.41258758306503296,
+      "memory(GiB)": 128.42,
+      "step": 191,
+      "token_acc": 0.8530646515533166,
+      "train_speed(iter/s)": 0.227723
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 1.5709357605507697,
+      "learning_rate": 9.281658496220272e-06,
+      "loss": 0.3878024220466614,
+      "memory(GiB)": 128.42,
+      "step": 192,
+      "token_acc": 0.8767885532591415,
+      "train_speed(iter/s)": 0.227734
+    },
+    {
+      "epoch": 0.8616071428571429,
+      "grad_norm": 1.5758516203486068,
+      "learning_rate": 9.272097022732444e-06,
+      "loss": 0.3386090099811554,
+      "memory(GiB)": 128.42,
+      "step": 193,
+      "token_acc": 0.8722772277227723,
+      "train_speed(iter/s)": 0.22775
+    },
+    {
+      "epoch": 0.8660714285714286,
+      "grad_norm": 1.7875878983881768,
+      "learning_rate": 9.262477328032972e-06,
+      "loss": 0.36395904421806335,
+      "memory(GiB)": 128.42,
+      "step": 194,
+      "token_acc": 0.8647058823529412,
+      "train_speed(iter/s)": 0.227755
+    },
+    {
+      "epoch": 0.8705357142857143,
+      "grad_norm": 1.7206122039653617,
+      "learning_rate": 9.252799543221478e-06,
+      "loss": 0.3222510814666748,
+      "memory(GiB)": 128.42,
+      "step": 195,
+      "token_acc": 0.8872691933916423,
+      "train_speed(iter/s)": 0.227771
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 1.6063432393863786,
+      "learning_rate": 9.24306380018924e-06,
+      "loss": 0.366471529006958,
+      "memory(GiB)": 128.42,
+      "step": 196,
+      "token_acc": 0.8529680365296803,
+      "train_speed(iter/s)": 0.227755
+    },
+    {
+      "epoch": 0.8794642857142857,
+      "grad_norm": 1.5262897164552471,
+      "learning_rate": 9.23327023161741e-06,
+      "loss": 0.35482266545295715,
+      "memory(GiB)": 128.42,
+      "step": 197,
+      "token_acc": 0.8913502109704642,
+      "train_speed(iter/s)": 0.227775
+    },
+    {
+      "epoch": 0.8839285714285714,
+      "grad_norm": 1.5528440615685493,
+      "learning_rate": 9.223418970975206e-06,
+      "loss": 0.34359997510910034,
+      "memory(GiB)": 128.42,
+      "step": 198,
+      "token_acc": 0.8822222222222222,
+      "train_speed(iter/s)": 0.227794
+    },
+    {
+      "epoch": 0.8883928571428571,
+      "grad_norm": 1.6542393490950473,
+      "learning_rate": 9.213510152518079e-06,
+      "loss": 0.35520508885383606,
+      "memory(GiB)": 128.42,
+      "step": 199,
+      "token_acc": 0.8734087694483734,
+      "train_speed(iter/s)": 0.227806
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 1.6709892296463946,
+      "learning_rate": 9.203543911285897e-06,
+      "loss": 0.41599422693252563,
+      "memory(GiB)": 128.42,
+      "step": 200,
+      "token_acc": 0.852233676975945,
+      "train_speed(iter/s)": 0.227806
+    },
+    {
+      "epoch": 0.8973214285714286,
+      "grad_norm": 1.671640006565075,
+      "learning_rate": 9.1935203831011e-06,
+      "loss": 0.35883253812789917,
+      "memory(GiB)": 128.42,
+      "step": 201,
+      "token_acc": 0.8736122971818958,
+      "train_speed(iter/s)": 0.22781
+    },
+    {
+      "epoch": 0.9017857142857143,
+      "grad_norm": 1.6477395677283788,
+      "learning_rate": 9.183439704566851e-06,
+      "loss": 0.37051939964294434,
+      "memory(GiB)": 128.42,
+      "step": 202,
+      "token_acc": 0.8587786259541985,
+      "train_speed(iter/s)": 0.227826
+    },
+    {
+      "epoch": 0.90625,
+      "grad_norm": 1.853741854901314,
+      "learning_rate": 9.173302013065168e-06,
+      "loss": 0.4104780852794647,
+      "memory(GiB)": 128.42,
+      "step": 203,
+      "token_acc": 0.8921810699588477,
+      "train_speed(iter/s)": 0.227845
+    },
+    {
+      "epoch": 0.9107142857142857,
+      "grad_norm": 1.5402394296023612,
+      "learning_rate": 9.163107446755058e-06,
+      "loss": 0.3700962960720062,
+      "memory(GiB)": 128.42,
+      "step": 204,
+      "token_acc": 0.8727615457115928,
+      "train_speed(iter/s)": 0.227855
+    },
+    {
+      "epoch": 0.9151785714285714,
+      "grad_norm": 1.7657060068556503,
+      "learning_rate": 9.152856144570633e-06,
+      "loss": 0.40010184049606323,
+      "memory(GiB)": 128.42,
+      "step": 205,
+      "token_acc": 0.8792415169660679,
+      "train_speed(iter/s)": 0.227847
+    },
+    {
+      "epoch": 0.9196428571428571,
+      "grad_norm": 1.9095214293712117,
+      "learning_rate": 9.142548246219212e-06,
+      "loss": 0.41824835538864136,
+      "memory(GiB)": 128.42,
+      "step": 206,
+      "token_acc": 0.8832159624413145,
+      "train_speed(iter/s)": 0.227836
+    },
+    {
+      "epoch": 0.9241071428571429,
+      "grad_norm": 1.7097757171587031,
+      "learning_rate": 9.132183892179428e-06,
+      "loss": 0.3989163935184479,
+      "memory(GiB)": 128.42,
+      "step": 207,
+      "token_acc": 0.8699690402476781,
+      "train_speed(iter/s)": 0.227851
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 1.7525425330091031,
+      "learning_rate": 9.121763223699297e-06,
+      "loss": 0.384308785200119,
+      "memory(GiB)": 128.42,
+      "step": 208,
+      "token_acc": 0.8719211822660099,
+      "train_speed(iter/s)": 0.227863
+    },
+    {
+      "epoch": 0.9330357142857143,
+      "grad_norm": 1.687286542996637,
+      "learning_rate": 9.111286382794313e-06,
+      "loss": 0.39490386843681335,
+      "memory(GiB)": 128.42,
+      "step": 209,
+      "token_acc": 0.8538461538461538,
+      "train_speed(iter/s)": 0.227879
+    },
+    {
+      "epoch": 0.9375,
+      "grad_norm": 1.6728271677101256,
+      "learning_rate": 9.100753512245493e-06,
+      "loss": 0.40762174129486084,
+      "memory(GiB)": 128.42,
+      "step": 210,
+      "token_acc": 0.8591439688715953,
+      "train_speed(iter/s)": 0.227893
+    },
+    {
+      "epoch": 0.9419642857142857,
+      "grad_norm": 1.5297271530329113,
+      "learning_rate": 9.090164755597444e-06,
+      "loss": 0.33648815751075745,
+      "memory(GiB)": 128.42,
+      "step": 211,
+      "token_acc": 0.8709454796411318,
+      "train_speed(iter/s)": 0.227904
+    },
+    {
+      "epoch": 0.9464285714285714,
+      "grad_norm": 1.6329682526256801,
+      "learning_rate": 9.079520257156402e-06,
+      "loss": 0.3839446008205414,
+      "memory(GiB)": 128.42,
+      "step": 212,
+      "token_acc": 0.8720073664825047,
+      "train_speed(iter/s)": 0.227907
+    },
+    {
+      "epoch": 0.9508928571428571,
+      "grad_norm": 1.6995622246856035,
+      "learning_rate": 9.068820161988268e-06,
+      "loss": 0.37411996722221375,
+      "memory(GiB)": 128.42,
+      "step": 213,
+      "token_acc": 0.8675246025738077,
+      "train_speed(iter/s)": 0.22791
+    },
+    {
+      "epoch": 0.9553571428571429,
+      "grad_norm": 1.644210658432108,
+      "learning_rate": 9.058064615916623e-06,
+      "loss": 0.40447884798049927,
+      "memory(GiB)": 128.42,
+      "step": 214,
+      "token_acc": 0.8810900082576383,
+      "train_speed(iter/s)": 0.227909
+    },
+    {
+      "epoch": 0.9598214285714286,
+      "grad_norm": 1.73954688053968,
+      "learning_rate": 9.047253765520758e-06,
+      "loss": 0.3625633716583252,
+      "memory(GiB)": 128.42,
+      "step": 215,
+      "token_acc": 0.8854718981972428,
+      "train_speed(iter/s)": 0.227913
+    },
+    {
+      "epoch": 0.9642857142857143,
+      "grad_norm": 1.5014266406948364,
+      "learning_rate": 9.036387758133653e-06,
+      "loss": 0.35661768913269043,
+      "memory(GiB)": 128.42,
+      "step": 216,
+      "token_acc": 0.8856275303643725,
+      "train_speed(iter/s)": 0.227927
+    },
+    {
+      "epoch": 0.96875,
+      "grad_norm": 1.4981188941125985,
+      "learning_rate": 9.025466741839987e-06,
+      "loss": 0.3550759553909302,
+      "memory(GiB)": 128.42,
+      "step": 217,
+      "token_acc": 0.9011786038077969,
+      "train_speed(iter/s)": 0.227946
+    },
+    {
+      "epoch": 0.9732142857142857,
+      "grad_norm": 1.5386161305286659,
+      "learning_rate": 9.014490865474119e-06,
+      "loss": 0.36008650064468384,
+      "memory(GiB)": 128.42,
+      "step": 218,
+      "token_acc": 0.8419429452582884,
+      "train_speed(iter/s)": 0.227967
+    },
+    {
+      "epoch": 0.9776785714285714,
+      "grad_norm": 1.612365460152363,
+      "learning_rate": 9.003460278618046e-06,
+      "loss": 0.3536672592163086,
+      "memory(GiB)": 128.42,
+      "step": 219,
+      "token_acc": 0.8841059602649006,
+      "train_speed(iter/s)": 0.227984
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 1.5788262432034628,
+      "learning_rate": 8.992375131599376e-06,
+      "loss": 0.3935646712779999,
+      "memory(GiB)": 128.42,
+      "step": 220,
+      "token_acc": 0.8902691511387164,
+      "train_speed(iter/s)": 0.227957
+    },
+    {
+      "epoch": 0.9866071428571429,
+      "grad_norm": 1.642418411618006,
+      "learning_rate": 8.981235575489282e-06,
+      "loss": 0.36323970556259155,
+      "memory(GiB)": 128.42,
+      "step": 221,
+      "token_acc": 0.8784956605593057,
+      "train_speed(iter/s)": 0.227976
+    },
+    {
+      "epoch": 0.9910714285714286,
+      "grad_norm": 1.7315886339339615,
+      "learning_rate": 8.97004176210043e-06,
+      "loss": 0.3866032063961029,
+      "memory(GiB)": 128.42,
+      "step": 222,
+      "token_acc": 0.8881188118811881,
+      "train_speed(iter/s)": 0.22799
+    },
+    {
+      "epoch": 0.9955357142857143,
+      "grad_norm": 1.5614784501414387,
+      "learning_rate": 8.958793843984923e-06,
+      "loss": 0.3766946792602539,
+      "memory(GiB)": 128.42,
+      "step": 223,
+      "token_acc": 0.8715124816446402,
+      "train_speed(iter/s)": 0.228007
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.5980875576332965,
+      "learning_rate": 8.947491974432216e-06,
+      "loss": 0.3782101273536682,
+      "memory(GiB)": 128.42,
+      "step": 224,
+      "token_acc": 0.9090909090909091,
+      "train_speed(iter/s)": 0.228013
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.36971455812454224,
+      "eval_runtime": 1.9584,
+      "eval_samples_per_second": 2.042,
+      "eval_steps_per_second": 0.511,
+      "eval_token_acc": 0.8700199203187251,
+      "step": 224
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 896,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 114578270584832.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

v0-20251012-211828/checkpoint-224/training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:88b1c4a5028e83060e213b58a43474a92ff6a3042e8f34900d9a7ab9f5639e64
+size 8312

v0-20251012-211828/checkpoint-224/vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff

v0-20251012-211828/checkpoint-224/zero_to_fp32.py ADDED Viewed

	@@ -0,0 +1,604 @@

+#!/usr/bin/env python
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+# DeepSpeed Team
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+debug = 0
+# load to cpu
+device = torch.device('cpu')
+def atoi(text):
+    return int(text) if text.isdigit() else text
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+    return file
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+    return ckpt_files
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+        ds_version = state_dict.get(DS_VERSION, None)
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+    return zero_model_states
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+    if type(world_size) is list:
+        world_size = max(world_size)
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+    return zero_stage, world_size, fp32_flat_groups
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+    model_files = get_model_state_files(ds_checkpoint_dir)
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        state_dict[name] = frozen_param_fragments[name]
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+    return state_dict
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+    offset *= world_size
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+    return state_dict
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    Returns:
+        - pytorch ``state_dict``
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+    A typical usage might be ::
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+    Returns:
+        - ``model`: modified model
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+    A typical usage might be ::
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+    return model
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+    debug = args.debug
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

v0-20251012-211828/checkpoint-448/added_tokens.json ADDED Viewed

	@@ -0,0 +1,24 @@

+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

v0-20251012-211828/checkpoint-448/args.json ADDED Viewed

	@@ -0,0 +1,369 @@

+{
+  "model": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "model_type": "qwen2_5",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": "flash_attn",
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "response_prefix": null,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "use_chat_template": true,
+  "template_backend": "swift",
+  "dataset": [
+    "/data/minimax-dialogue/users/ruobai/cogito/base_dataset/broswer_training_data/merged_data.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "data_seed": 42,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": false,
+  "model_name": [
+    null,
+    null
+  ],
+  "model_author": [
+    null,
+    null
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "full",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": true,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ddp_timeout": 1800,
+  "ddp_backend": "nccl",
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 2,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 1e-05,
+  "weight_decay": 0.05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 4.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 20.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "wandb"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": false,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 8,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": true,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "optimizer": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 8,
+  "local_world_size": 8,
+  "model_suffix": "Qwen2.5-7B",
+  "model_info": "ModelInfo(model_type='qwen2_5', model_dir='/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B', torch_dtype=torch.bfloat16, max_model_len=131072, quant_method=None, quant_bits=None, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct', hf_model_id='Qwen/Qwen2.5-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct', hf_model_id='Qwen/Qwen2.5-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct', hf_model_id='Qwen/Qwen2.5-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct', hf_model_id='Qwen/Qwen2.5-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct', hf_model_id='Qwen/Qwen2.5-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B', hf_model_id='Qwen/Qwen2.5-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B', hf_model_id='Qwen/Qwen2.5-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B', hf_model_id='Qwen/Qwen2.5-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B', hf_model_id='Qwen/Qwen2.5-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B', hf_model_id='Qwen/Qwen2.5-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B', hf_model_id='Qwen/Qwen2.5-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B', hf_model_id='Qwen/Qwen2.5-72B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B', hf_model_id='Qwen/Qwen2.5-Coder-0.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B', hf_model_id='Qwen/Qwen2.5-Coder-1.5B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B', hf_model_id='Qwen/Qwen2.5-Coder-3B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B', hf_model_id='Qwen/Qwen2.5-Coder-7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B', hf_model_id='Qwen/Qwen2.5-Coder-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B', hf_model_id='Qwen/Qwen2.5-Coder-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', hf_model_id='Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=['coding'])], template='qwen2_5', get_function=<function get_model_tokenizer_with_flash_attn at 0x7f3ff78a7920>, model_arch='llama', architectures=['Qwen2ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.37'], tags=[])",
+  "model_dir": "/data/minimax-dialogue/users/ruobai/cogito/base_model/Qwen2.5-7B",
+  "hub": "<class 'swift.hub.hub.HFHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=2, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.05, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=4.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend='nccl', tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=20.0, dataloader_num_workers=48, dataloader_prefetch_factor=10, past_index=-1, run_name='/data/minimax-dialogue/users/ruobai/swift/model/qwen2p5_7B_ba_1012/v0-20251012-211828', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['wandb'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='full', optimizer=None, local_repo_path=None, galore_config=None)"
+}

v0-20251012-211828/checkpoint-448/config.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

v0-20251012-211828/checkpoint-448/generation_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.51.3"
+}

v0-20251012-211828/checkpoint-448/latest ADDED Viewed

	@@ -0,0 +1 @@


1	+ global_step448

v0-20251012-211828/checkpoint-448/merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

v0-20251012-211828/checkpoint-448/model-00001-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9ae082b6d4a3335bddf2e3ad359953f59cf7ba12511dd539b78be26d70a8c6c5
+size 4877660776

v0-20251012-211828/checkpoint-448/model-00002-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2e4a54c18739b011f1f716af6aae8494c22c946c0c2578c4f2003d37fa0fb027
+size 4932751008

v0-20251012-211828/checkpoint-448/model-00003-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:628b3a445a516905e16572aba05128d1a8d7a9f2d549f373ebffcf0b66f2dc11
+size 4330865200

v0-20251012-211828/checkpoint-448/model-00004-of-00004.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:cadea09cd03a13198884d136be0b7fe07bcbdea6787fb749cee4c5657cec7d9c
+size 1089994880

v0-20251012-211828/checkpoint-448/model.safetensors.index.json ADDED Viewed

	@@ -0,0 +1,346 @@

+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

v0-20251012-211828/checkpoint-448/rng_state_0.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff22c72cb50d4b6353cf336b950d2c54115c739606544de9a8d3b0fab0ef188
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_1.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c253ad05d3ffa98dc1e5291ec640e6158218602254206e6fb97ea82185040775
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_2.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:edcdbb53b869133655de3ceb14325bb161e7aa238640244125393ff4afcb7363
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_3.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b095e9807a468e05241a1d389fd326265bee89778655d3c00298c2f6abcee791
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_4.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a4da3805c54028ef89f9cce32b640836c54b432b251fae2dea3d0182fa96f403
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_5.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:08bb3c8d2c1053a6b93a1342acb589ea4695b3fdb4dfeaf3675191c7e3390d3d
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_6.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3cfebda0f5a673869a40a8a82d7e308c843f9b55ec287861a6a0670b4c43c9d4
+size 15920

v0-20251012-211828/checkpoint-448/rng_state_7.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:34bc6aae481e0bf889f93cc019a7692fbdaba7925f341ec202dd711aa1d2a724
+size 15920

v0-20251012-211828/checkpoint-448/scheduler.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0c199a071e8f3f4409364ebb7ec6e95774df918d3f419a206c18e162e774e54b
+size 1064

v0-20251012-211828/checkpoint-448/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,31 @@

+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}