Upload folder using huggingface_hub

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/args.json

@@ -0,0 +1,403 @@
+{
+  "output_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 5e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 2.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.2,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 2000.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": [],
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "project": "huggingface",
+  "trackio_space_id": "trackio",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": true,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "lora",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "model_type": "qwen2_5_vl",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5_vl",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "data.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": true,
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": false,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 16,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "Qwen2.5-VL-7B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2_5_vl', model_dir='/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=128000, quant_method=None, quant_bits=None, rope_scaling={'type': 'default', 'mrope_section': [16, 24, 24], 'rope_type': 'default'}, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5_vl', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', hf_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-RL', hf_model_id='XiaomiMiMo/MiMo-VL-7B-RL', model_path=None, ms_revision=None, hf_revision=None)], template='mimo_vl', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.models.qwen.Qwen2_5VLLoader'>, template='qwen2_5_vl', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), mcore_model_type=None, architectures=['Qwen2_5_VLForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'], tags=[])",
+  "model_dir": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "template_meta": "QwenTemplateMeta(template_type='qwen2_5_vl', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.templates.qwen.Qwen2_5VLTemplate'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": true,
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=5e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.2, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=2, past_index=-1, run_name='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], project='huggingface', trackio_space_id='trackio', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='lora', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/adapter_config.json

@@ -0,0 +1,40 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [],
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "^(model\\.language_model(?=\\.).*\\.(up_proj|o_proj|down_proj|q_proj|v_proj|gate_proj|k_proj)|(?!(model.visual.merger))model\\.visual(?=\\.).*\\.(qkv|up_proj|mlp.2|down_proj|mlp.0|attn.proj|gate_proj))$",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d95481f2456b40bbce0c8ed48286384ee4f90642b04060131c9b20f7447c4d1
+size 103145560

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/additional_config.json

@@ -0,0 +1 @@
+{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/args.json

@@ -0,0 +1,403 @@
+{
+  "output_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 5e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 2.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.2,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 2000.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": [],
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "project": "huggingface",
+  "trackio_space_id": "trackio",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": true,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "lora",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "model_type": "qwen2_5_vl",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5_vl",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "data.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": true,
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": false,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 16,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "Qwen2.5-VL-7B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2_5_vl', model_dir='/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=128000, quant_method=None, quant_bits=None, rope_scaling={'type': 'default', 'mrope_section': [16, 24, 24], 'rope_type': 'default'}, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5_vl', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', hf_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-RL', hf_model_id='XiaomiMiMo/MiMo-VL-7B-RL', model_path=None, ms_revision=None, hf_revision=None)], template='mimo_vl', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.models.qwen.Qwen2_5VLLoader'>, template='qwen2_5_vl', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), mcore_model_type=None, architectures=['Qwen2_5_VLForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'], tags=[])",
+  "model_dir": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "template_meta": "QwenTemplateMeta(template_type='qwen2_5_vl', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.templates.qwen.Qwen2_5VLTemplate'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": true,
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=5e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.2, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=2, past_index=-1, run_name='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], project='huggingface', trackio_space_id='trackio', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='lora', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287e2bcefb6253ed8b45b79ad807f2697967afce025b2ba2d1c51a097ce5024c
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:799db15bb7cde0de8f4342bcbcf1a9e4b9d6973423d668b1923f680acff74800
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6443033bb25191b7d9a576f71fbcbdb94d2b50ae603d093fd14175d673b1ef59
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba54cb957fbe12c2b986bf9b17ebe6da85375d808cc06cf5c56365f1c55e431d
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee1d0bfff9a1227418e4a67d588529eb69aabf6ecc018d57b368c26eda4ef20
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccdc6f17819e84c4ce50109643efce192a115d4ed049838f9e626385a1094859
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2e8fa754c15642022fa305fc6ea93d522d6b74481587c164939d512b11b842a
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/global_step394/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6ebbfc364320d2d153672e6bd3deda1a7aa72c2854f22bd64c16403d008d5a
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/latest

@@ -0,0 +1 @@
+global_step394

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:896573e771d74e064376d38881cd4b078122797c17126dc4dfc997b621cc5ac6
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54c981c54ebaac3235bea9791ea4db035522184b0ae26ff26eeb7f3a4e5835f9
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1623a15e509683723159ea15c0ebe929e6c6579925f349f18eb0d280ef0ceea
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85851983cd092edf0ed4f6e2d0b855da520b1f9d044c8d96897eecf8ab218ae9
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5231310b50f521b1700af4c096de931cb9a32e5a4590c23558cb566596c5699
+size 1465

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/trainer_state.json

@@ -0,0 +1,3195 @@
+{
+  "best_global_step": 394,
+  "best_metric": 0.38288021,
+  "best_model_checkpoint": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394",
+  "epoch": 1.0,
+  "eval_steps": 2000.0,
+  "global_step": 394,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0025388765471278957,
+      "grad_norm": 3.644420246106802,
+      "learning_rate": 3.1645569620253163e-07,
+      "loss": 1.318245530128479,
+      "step": 1,
+      "token_acc": 0.7163964997848228
+    },
+    {
+      "epoch": 0.0050777530942557915,
+      "grad_norm": 7.066332979169105,
+      "learning_rate": 6.329113924050633e-07,
+      "loss": 1.3213859796524048,
+      "step": 2,
+      "token_acc": 0.7273305758829144
+    },
+    {
+      "epoch": 0.007616629641383688,
+      "grad_norm": 10.199557510236037,
+      "learning_rate": 9.493670886075951e-07,
+      "loss": 1.3628352880477905,
+      "step": 3,
+      "token_acc": 0.7068235294117647
+    },
+    {
+      "epoch": 0.010155506188511583,
+      "grad_norm": 2.8758103179283525,
+      "learning_rate": 1.2658227848101265e-06,
+      "loss": 1.365400791168213,
+      "step": 4,
+      "token_acc": 0.7117047560222359
+    },
+    {
+      "epoch": 0.012694382735639479,
+      "grad_norm": 3.297140385498391,
+      "learning_rate": 1.5822784810126583e-06,
+      "loss": 1.400863528251648,
+      "step": 5,
+      "token_acc": 0.698011137629276
+    },
+    {
+      "epoch": 0.015233259282767376,
+      "grad_norm": 5.493150648764733,
+      "learning_rate": 1.8987341772151901e-06,
+      "loss": 1.35938560962677,
+      "step": 6,
+      "token_acc": 0.7149187592319055
+    },
+    {
+      "epoch": 0.017772135829895272,
+      "grad_norm": 8.171500030817667,
+      "learning_rate": 2.2151898734177215e-06,
+      "loss": 1.3011455535888672,
+      "step": 7,
+      "token_acc": 0.7231049321416749
+    },
+    {
+      "epoch": 0.020311012377023166,
+      "grad_norm": 5.827210745414414,
+      "learning_rate": 2.531645569620253e-06,
+      "loss": 1.417736291885376,
+      "step": 8,
+      "token_acc": 0.714411093558013
+    },
+    {
+      "epoch": 0.022849888924151063,
+      "grad_norm": 6.1293874585644526,
+      "learning_rate": 2.848101265822785e-06,
+      "loss": 1.378328800201416,
+      "step": 9,
+      "token_acc": 0.7031924072476272
+    },
+    {
+      "epoch": 0.025388765471278957,
+      "grad_norm": 3.520366609215098,
+      "learning_rate": 3.1645569620253167e-06,
+      "loss": 1.343224287033081,
+      "step": 10,
+      "token_acc": 0.7227991691963572
+    },
+    {
+      "epoch": 0.027927642018406855,
+      "grad_norm": 3.6585128819526824,
+      "learning_rate": 3.4810126582278482e-06,
+      "loss": 1.390702724456787,
+      "step": 11,
+      "token_acc": 0.7076406649616368
+    },
+    {
+      "epoch": 0.030466518565534752,
+      "grad_norm": 6.217581279052289,
+      "learning_rate": 3.7974683544303802e-06,
+      "loss": 1.3524142503738403,
+      "step": 12,
+      "token_acc": 0.7028127313101407
+    },
+    {
+      "epoch": 0.03300539511266265,
+      "grad_norm": 3.388086422416274,
+      "learning_rate": 4.113924050632911e-06,
+      "loss": 1.3935208320617676,
+      "step": 13,
+      "token_acc": 0.7046906654199782
+    },
+    {
+      "epoch": 0.035544271659790544,
+      "grad_norm": 8.332931037303998,
+      "learning_rate": 4.430379746835443e-06,
+      "loss": 1.3878122568130493,
+      "step": 14,
+      "token_acc": 0.7046968687541639
+    },
+    {
+      "epoch": 0.03808314820691844,
+      "grad_norm": 17.896380479546966,
+      "learning_rate": 4.746835443037975e-06,
+      "loss": 1.2754666805267334,
+      "step": 15,
+      "token_acc": 0.7242310577644411
+    },
+    {
+      "epoch": 0.04062202475404633,
+      "grad_norm": 11.46365102116194,
+      "learning_rate": 5.063291139240506e-06,
+      "loss": 1.3468568325042725,
+      "step": 16,
+      "token_acc": 0.7084963229541543
+    },
+    {
+      "epoch": 0.04316090130117423,
+      "grad_norm": 4.2821219745746655,
+      "learning_rate": 5.379746835443038e-06,
+      "loss": 1.40608549118042,
+      "step": 17,
+      "token_acc": 0.6968503937007874
+    },
+    {
+      "epoch": 0.04569977784830213,
+      "grad_norm": 13.278823682877052,
+      "learning_rate": 5.69620253164557e-06,
+      "loss": 1.300422191619873,
+      "step": 18,
+      "token_acc": 0.7242131505141789
+    },
+    {
+      "epoch": 0.04823865439543002,
+      "grad_norm": 2.778568201618841,
+      "learning_rate": 6.012658227848101e-06,
+      "loss": 1.3542265892028809,
+      "step": 19,
+      "token_acc": 0.7110212691158376
+    },
+    {
+      "epoch": 0.050777530942557915,
+      "grad_norm": 10.575315873473542,
+      "learning_rate": 6.329113924050633e-06,
+      "loss": 1.3549412488937378,
+      "step": 20,
+      "token_acc": 0.7120073608342279
+    },
+    {
+      "epoch": 0.053316407489685816,
+      "grad_norm": 4.651761760848563,
+      "learning_rate": 6.6455696202531645e-06,
+      "loss": 1.3315010070800781,
+      "step": 21,
+      "token_acc": 0.7172937521574042
+    },
+    {
+      "epoch": 0.05585528403681371,
+      "grad_norm": 11.039499603303584,
+      "learning_rate": 6.9620253164556965e-06,
+      "loss": 1.3607463836669922,
+      "step": 22,
+      "token_acc": 0.7119058535839828
+    },
+    {
+      "epoch": 0.058394160583941604,
+      "grad_norm": 21.786442056735186,
+      "learning_rate": 7.2784810126582285e-06,
+      "loss": 1.3679276704788208,
+      "step": 23,
+      "token_acc": 0.7060693204177387
+    },
+    {
+      "epoch": 0.060933037131069505,
+      "grad_norm": 3.8926457068624027,
+      "learning_rate": 7.5949367088607605e-06,
+      "loss": 1.230813980102539,
+      "step": 24,
+      "token_acc": 0.7303050847457627
+    },
+    {
+      "epoch": 0.06347191367819739,
+      "grad_norm": 27.273056033594486,
+      "learning_rate": 7.911392405063292e-06,
+      "loss": 1.2661793231964111,
+      "step": 25,
+      "token_acc": 0.7191495088144261
+    },
+    {
+      "epoch": 0.0660107902253253,
+      "grad_norm": 17.170751148909975,
+      "learning_rate": 8.227848101265822e-06,
+      "loss": 1.3003311157226562,
+      "step": 26,
+      "token_acc": 0.7185164292765477
+    },
+    {
+      "epoch": 0.0685496667724532,
+      "grad_norm": 4.4377062155304925,
+      "learning_rate": 8.544303797468354e-06,
+      "loss": 1.3472293615341187,
+      "step": 27,
+      "token_acc": 0.7084078711985689
+    },
+    {
+      "epoch": 0.07108854331958109,
+      "grad_norm": 28.824459578387078,
+      "learning_rate": 8.860759493670886e-06,
+      "loss": 1.2210255861282349,
+      "step": 28,
+      "token_acc": 0.7316366961292289
+    },
+    {
+      "epoch": 0.07362741986670898,
+      "grad_norm": 6.453710147572056,
+      "learning_rate": 9.177215189873418e-06,
+      "loss": 1.3194985389709473,
+      "step": 29,
+      "token_acc": 0.7122530736289543
+    },
+    {
+      "epoch": 0.07616629641383688,
+      "grad_norm": 5.834087298103824,
+      "learning_rate": 9.49367088607595e-06,
+      "loss": 1.2912172079086304,
+      "step": 30,
+      "token_acc": 0.7186076772934288
+    },
+    {
+      "epoch": 0.07870517296096477,
+      "grad_norm": 2.871547428905148,
+      "learning_rate": 9.81012658227848e-06,
+      "loss": 1.315631628036499,
+      "step": 31,
+      "token_acc": 0.7080590902442592
+    },
+    {
+      "epoch": 0.08124404950809266,
+      "grad_norm": 3.6221352179116657,
+      "learning_rate": 1.0126582278481012e-05,
+      "loss": 1.2459921836853027,
+      "step": 32,
+      "token_acc": 0.7226173541963016
+    },
+    {
+      "epoch": 0.08378292605522057,
+      "grad_norm": 8.000389558921656,
+      "learning_rate": 1.0443037974683544e-05,
+      "loss": 1.271647334098816,
+      "step": 33,
+      "token_acc": 0.7218111867416396
+    },
+    {
+      "epoch": 0.08632180260234847,
+      "grad_norm": 4.361540191205865,
+      "learning_rate": 1.0759493670886076e-05,
+      "loss": 1.274230718612671,
+      "step": 34,
+      "token_acc": 0.7135694646611677
+    },
+    {
+      "epoch": 0.08886067914947636,
+      "grad_norm": 4.9147142745551164,
+      "learning_rate": 1.1075949367088608e-05,
+      "loss": 1.2685215473175049,
+      "step": 35,
+      "token_acc": 0.7207676044885347
+    },
+    {
+      "epoch": 0.09139955569660425,
+      "grad_norm": 3.195151105054266,
+      "learning_rate": 1.139240506329114e-05,
+      "loss": 1.299472451210022,
+      "step": 36,
+      "token_acc": 0.7125310717941219
+    },
+    {
+      "epoch": 0.09393843224373215,
+      "grad_norm": 2.6222925030737736,
+      "learning_rate": 1.170886075949367e-05,
+      "loss": 1.184287428855896,
+      "step": 37,
+      "token_acc": 0.7353410097431355
+    },
+    {
+      "epoch": 0.09647730879086004,
+      "grad_norm": 2.8207306382492585,
+      "learning_rate": 1.2025316455696203e-05,
+      "loss": 1.2705498933792114,
+      "step": 38,
+      "token_acc": 0.7183790421612771
+    },
+    {
+      "epoch": 0.09901618533798794,
+      "grad_norm": 4.451363242439522,
+      "learning_rate": 1.2341772151898735e-05,
+      "loss": 1.1410255432128906,
+      "step": 39,
+      "token_acc": 0.7343029443064917
+    },
+    {
+      "epoch": 0.10155506188511583,
+      "grad_norm": 5.113673628483296,
+      "learning_rate": 1.2658227848101267e-05,
+      "loss": 1.1782591342926025,
+      "step": 40,
+      "token_acc": 0.7327224724336077
+    },
+    {
+      "epoch": 0.10409393843224374,
+      "grad_norm": 3.5593986457435083,
+      "learning_rate": 1.2974683544303799e-05,
+      "loss": 1.1586289405822754,
+      "step": 41,
+      "token_acc": 0.7429899957728617
+    },
+    {
+      "epoch": 0.10663281497937163,
+      "grad_norm": 2.9195256626560684,
+      "learning_rate": 1.3291139240506329e-05,
+      "loss": 1.139272689819336,
+      "step": 42,
+      "token_acc": 0.7421777221526908
+    },
+    {
+      "epoch": 0.10917169152649953,
+      "grad_norm": 20.803104301053065,
+      "learning_rate": 1.3607594936708861e-05,
+      "loss": 1.1704292297363281,
+      "step": 43,
+      "token_acc": 0.7285796434732605
+    },
+    {
+      "epoch": 0.11171056807362742,
+      "grad_norm": 3.471589836187823,
+      "learning_rate": 1.3924050632911393e-05,
+      "loss": 1.1391760110855103,
+      "step": 44,
+      "token_acc": 0.7379228837346727
+    },
+    {
+      "epoch": 0.11424944462075531,
+      "grad_norm": 5.205417449479473,
+      "learning_rate": 1.4240506329113925e-05,
+      "loss": 1.1042394638061523,
+      "step": 45,
+      "token_acc": 0.7448979591836735
+    },
+    {
+      "epoch": 0.11678832116788321,
+      "grad_norm": 4.3574672102801015,
+      "learning_rate": 1.4556962025316457e-05,
+      "loss": 1.0657453536987305,
+      "step": 46,
+      "token_acc": 0.7489454347530277
+    },
+    {
+      "epoch": 0.1193271977150111,
+      "grad_norm": 23.053778423920146,
+      "learning_rate": 1.4873417721518987e-05,
+      "loss": 1.1951930522918701,
+      "step": 47,
+      "token_acc": 0.7246397287369314
+    },
+    {
+      "epoch": 0.12186607426213901,
+      "grad_norm": 3.803629904444073,
+      "learning_rate": 1.5189873417721521e-05,
+      "loss": 1.0914082527160645,
+      "step": 48,
+      "token_acc": 0.745520801700577
+    },
+    {
+      "epoch": 0.1244049508092669,
+      "grad_norm": 6.689336514482712,
+      "learning_rate": 1.550632911392405e-05,
+      "loss": 1.1104226112365723,
+      "step": 49,
+      "token_acc": 0.7371739753511035
+    },
+    {
+      "epoch": 0.12694382735639478,
+      "grad_norm": 13.295611456338577,
+      "learning_rate": 1.5822784810126583e-05,
+      "loss": 1.114711880683899,
+      "step": 50,
+      "token_acc": 0.734084314729476
+    },
+    {
+      "epoch": 0.12948270390352268,
+      "grad_norm": 3.6818930607365474,
+      "learning_rate": 1.6139240506329115e-05,
+      "loss": 1.0886123180389404,
+      "step": 51,
+      "token_acc": 0.7460102378801566
+    },
+    {
+      "epoch": 0.1320215804506506,
+      "grad_norm": 2.068572818027539,
+      "learning_rate": 1.6455696202531644e-05,
+      "loss": 0.9962466359138489,
+      "step": 52,
+      "token_acc": 0.7619711761971176
+    },
+    {
+      "epoch": 0.1345604569977785,
+      "grad_norm": 2.3029328438465955,
+      "learning_rate": 1.677215189873418e-05,
+      "loss": 1.048902153968811,
+      "step": 53,
+      "token_acc": 0.7497123130034522
+    },
+    {
+      "epoch": 0.1370993335449064,
+      "grad_norm": 5.7047860384924505,
+      "learning_rate": 1.7088607594936708e-05,
+      "loss": 1.0195245742797852,
+      "step": 54,
+      "token_acc": 0.7620052770448549
+    },
+    {
+      "epoch": 0.13963821009203428,
+      "grad_norm": 6.477061508345248,
+      "learning_rate": 1.7405063291139243e-05,
+      "loss": 1.081801176071167,
+      "step": 55,
+      "token_acc": 0.7465539422900203
+    },
+    {
+      "epoch": 0.14217708663916218,
+      "grad_norm": 6.2376824743665855,
+      "learning_rate": 1.7721518987341772e-05,
+      "loss": 1.0337473154067993,
+      "step": 56,
+      "token_acc": 0.7620926243567753
+    },
+    {
+      "epoch": 0.14471596318629007,
+      "grad_norm": 2.5584859222177037,
+      "learning_rate": 1.8037974683544304e-05,
+      "loss": 1.0235919952392578,
+      "step": 57,
+      "token_acc": 0.7479560191711305
+    },
+    {
+      "epoch": 0.14725483973341796,
+      "grad_norm": 1.6007308463984253,
+      "learning_rate": 1.8354430379746836e-05,
+      "loss": 1.026627540588379,
+      "step": 58,
+      "token_acc": 0.7545010680500458
+    },
+    {
+      "epoch": 0.14979371628054586,
+      "grad_norm": 3.5256130287772764,
+      "learning_rate": 1.8670886075949368e-05,
+      "loss": 1.0003381967544556,
+      "step": 59,
+      "token_acc": 0.7527782006393667
+    },
+    {
+      "epoch": 0.15233259282767375,
+      "grad_norm": 3.7023204963636838,
+      "learning_rate": 1.89873417721519e-05,
+      "loss": 0.9081876873970032,
+      "step": 60,
+      "token_acc": 0.7770368222673238
+    },
+    {
+      "epoch": 0.15487146937480165,
+      "grad_norm": 13.207752417441773,
+      "learning_rate": 1.9303797468354432e-05,
+      "loss": 1.0521725416183472,
+      "step": 61,
+      "token_acc": 0.7498269417139692
+    },
+    {
+      "epoch": 0.15741034592192954,
+      "grad_norm": 5.75860602043051,
+      "learning_rate": 1.962025316455696e-05,
+      "loss": 0.9959630966186523,
+      "step": 62,
+      "token_acc": 0.7539973595423207
+    },
+    {
+      "epoch": 0.15994922246905743,
+      "grad_norm": 2.1358911014473065,
+      "learning_rate": 1.9936708860759496e-05,
+      "loss": 0.9936040639877319,
+      "step": 63,
+      "token_acc": 0.7573317125056982
+    },
+    {
+      "epoch": 0.16248809901618533,
+      "grad_norm": 3.6065860167804553,
+      "learning_rate": 2.0253164556962025e-05,
+      "loss": 0.9302588701248169,
+      "step": 64,
+      "token_acc": 0.7660311958405546
+    },
+    {
+      "epoch": 0.16502697556331322,
+      "grad_norm": 3.4010076126873536,
+      "learning_rate": 2.056962025316456e-05,
+      "loss": 0.9686568975448608,
+      "step": 65,
+      "token_acc": 0.7588071919310042
+    },
+    {
+      "epoch": 0.16756585211044114,
+      "grad_norm": 8.25772123801425,
+      "learning_rate": 2.088607594936709e-05,
+      "loss": 0.9752371907234192,
+      "step": 66,
+      "token_acc": 0.7534704800133801
+    },
+    {
+      "epoch": 0.17010472865756904,
+      "grad_norm": 2.759694488314889,
+      "learning_rate": 2.120253164556962e-05,
+      "loss": 0.8781665563583374,
+      "step": 67,
+      "token_acc": 0.7766960284345542
+    },
+    {
+      "epoch": 0.17264360520469693,
+      "grad_norm": 6.065387925549159,
+      "learning_rate": 2.1518987341772153e-05,
+      "loss": 0.8539150953292847,
+      "step": 68,
+      "token_acc": 0.7778531456672885
+    },
+    {
+      "epoch": 0.17518248175182483,
+      "grad_norm": 6.273759879651115,
+      "learning_rate": 2.1835443037974685e-05,
+      "loss": 0.8206325173377991,
+      "step": 69,
+      "token_acc": 0.7851107226107226
+    },
+    {
+      "epoch": 0.17772135829895272,
+      "grad_norm": 6.797725562475832,
+      "learning_rate": 2.2151898734177217e-05,
+      "loss": 0.899059534072876,
+      "step": 70,
+      "token_acc": 0.7692409240924093
+    },
+    {
+      "epoch": 0.1802602348460806,
+      "grad_norm": 14.741247422293133,
+      "learning_rate": 2.246835443037975e-05,
+      "loss": 0.9375818371772766,
+      "step": 71,
+      "token_acc": 0.7635605006954103
+    },
+    {
+      "epoch": 0.1827991113932085,
+      "grad_norm": 2.91554003314857,
+      "learning_rate": 2.278481012658228e-05,
+      "loss": 0.8030121326446533,
+      "step": 72,
+      "token_acc": 0.7937145989402521
+    },
+    {
+      "epoch": 0.1853379879403364,
+      "grad_norm": 3.256147486736755,
+      "learning_rate": 2.3101265822784813e-05,
+      "loss": 0.9064524173736572,
+      "step": 73,
+      "token_acc": 0.7655107017068545
+    },
+    {
+      "epoch": 0.1878768644874643,
+      "grad_norm": 2.189919100543407,
+      "learning_rate": 2.341772151898734e-05,
+      "loss": 0.7569831609725952,
+      "step": 74,
+      "token_acc": 0.7891252955082743
+    },
+    {
+      "epoch": 0.1904157410345922,
+      "grad_norm": 3.0168260923297563,
+      "learning_rate": 2.3734177215189873e-05,
+      "loss": 0.8783301115036011,
+      "step": 75,
+      "token_acc": 0.7689860284963342
+    },
+    {
+      "epoch": 0.19295461758172008,
+      "grad_norm": 14.13725393363498,
+      "learning_rate": 2.4050632911392405e-05,
+      "loss": 0.8693279027938843,
+      "step": 76,
+      "token_acc": 0.7832378223495702
+    },
+    {
+      "epoch": 0.19549349412884798,
+      "grad_norm": 4.663170307038689,
+      "learning_rate": 2.4367088607594937e-05,
+      "loss": 0.9367852807044983,
+      "step": 77,
+      "token_acc": 0.7587620685094564
+    },
+    {
+      "epoch": 0.19803237067597587,
+      "grad_norm": 2.0251332116596314,
+      "learning_rate": 2.468354430379747e-05,
+      "loss": 0.9105110764503479,
+      "step": 78,
+      "token_acc": 0.7696913251729643
+    },
+    {
+      "epoch": 0.20057124722310377,
+      "grad_norm": 2.9353487138799976,
+      "learning_rate": 2.5e-05,
+      "loss": 0.812760055065155,
+      "step": 79,
+      "token_acc": 0.7861037544788908
+    },
+    {
+      "epoch": 0.20311012377023166,
+      "grad_norm": 6.81850589494243,
+      "learning_rate": 2.5316455696202533e-05,
+      "loss": 0.8404111266136169,
+      "step": 80,
+      "token_acc": 0.7826737386100912
+    },
+    {
+      "epoch": 0.20564900031735958,
+      "grad_norm": 5.8636033139277925,
+      "learning_rate": 2.5632911392405062e-05,
+      "loss": 0.7662273049354553,
+      "step": 81,
+      "token_acc": 0.7912533814247069
+    },
+    {
+      "epoch": 0.20818787686448748,
+      "grad_norm": 3.1667411014534554,
+      "learning_rate": 2.5949367088607597e-05,
+      "loss": 0.8550683259963989,
+      "step": 82,
+      "token_acc": 0.7763946519133241
+    },
+    {
+      "epoch": 0.21072675341161537,
+      "grad_norm": 2.9382867326927786,
+      "learning_rate": 2.626582278481013e-05,
+      "loss": 0.728255033493042,
+      "step": 83,
+      "token_acc": 0.7997798049701164
+    },
+    {
+      "epoch": 0.21326562995874326,
+      "grad_norm": 4.413930575957828,
+      "learning_rate": 2.6582278481012658e-05,
+      "loss": 0.7953159213066101,
+      "step": 84,
+      "token_acc": 0.7851035843472542
+    },
+    {
+      "epoch": 0.21580450650587116,
+      "grad_norm": 2.2548226528829085,
+      "learning_rate": 2.689873417721519e-05,
+      "loss": 0.8843717575073242,
+      "step": 85,
+      "token_acc": 0.7768637532133676
+    },
+    {
+      "epoch": 0.21834338305299905,
+      "grad_norm": 13.172644883413941,
+      "learning_rate": 2.7215189873417722e-05,
+      "loss": 0.8480836749076843,
+      "step": 86,
+      "token_acc": 0.7769437756691687
+    },
+    {
+      "epoch": 0.22088225960012695,
+      "grad_norm": 13.962359276231266,
+      "learning_rate": 2.7531645569620257e-05,
+      "loss": 0.7050020098686218,
+      "step": 87,
+      "token_acc": 0.8056864831990269
+    },
+    {
+      "epoch": 0.22342113614725484,
+      "grad_norm": 4.2019275549330555,
+      "learning_rate": 2.7848101265822786e-05,
+      "loss": 0.7555727958679199,
+      "step": 88,
+      "token_acc": 0.7975503308461214
+    },
+    {
+      "epoch": 0.22596001269438273,
+      "grad_norm": 5.0153336369322,
+      "learning_rate": 2.8164556962025318e-05,
+      "loss": 0.7249046564102173,
+      "step": 89,
+      "token_acc": 0.8017741935483871
+    },
+    {
+      "epoch": 0.22849888924151063,
+      "grad_norm": 11.63320257189594,
+      "learning_rate": 2.848101265822785e-05,
+      "loss": 0.688890814781189,
+      "step": 90,
+      "token_acc": 0.8080682699767261
+    },
+    {
+      "epoch": 0.23103776578863852,
+      "grad_norm": 5.457398735429565,
+      "learning_rate": 2.879746835443038e-05,
+      "loss": 0.7510577440261841,
+      "step": 91,
+      "token_acc": 0.7953611274221961
+    },
+    {
+      "epoch": 0.23357664233576642,
+      "grad_norm": 4.530006374919538,
+      "learning_rate": 2.9113924050632914e-05,
+      "loss": 0.7740436792373657,
+      "step": 92,
+      "token_acc": 0.7910108958837773
+    },
+    {
+      "epoch": 0.2361155188828943,
+      "grad_norm": 3.283408990769055,
+      "learning_rate": 2.9430379746835446e-05,
+      "loss": 0.6994897127151489,
+      "step": 93,
+      "token_acc": 0.8084580953850903
+    },
+    {
+      "epoch": 0.2386543954300222,
+      "grad_norm": 3.74567677581687,
+      "learning_rate": 2.9746835443037974e-05,
+      "loss": 0.6745874285697937,
+      "step": 94,
+      "token_acc": 0.8121098626716604
+    },
+    {
+      "epoch": 0.2411932719771501,
+      "grad_norm": 2.481160635241608,
+      "learning_rate": 3.0063291139240506e-05,
+      "loss": 0.7404159307479858,
+      "step": 95,
+      "token_acc": 0.8046498830650709
+    },
+    {
+      "epoch": 0.24373214852427802,
+      "grad_norm": 3.011989315773918,
+      "learning_rate": 3.0379746835443042e-05,
+      "loss": 0.6883790493011475,
+      "step": 96,
+      "token_acc": 0.8137285986049461
+    },
+    {
+      "epoch": 0.2462710250714059,
+      "grad_norm": 3.977588680498373,
+      "learning_rate": 3.0696202531645574e-05,
+      "loss": 0.7354066371917725,
+      "step": 97,
+      "token_acc": 0.8012324221835994
+    },
+    {
+      "epoch": 0.2488099016185338,
+      "grad_norm": 2.4417231744019787,
+      "learning_rate": 3.10126582278481e-05,
+      "loss": 0.7102183103561401,
+      "step": 98,
+      "token_acc": 0.801501463290495
+    },
+    {
+      "epoch": 0.2513487781656617,
+      "grad_norm": 5.211297364565694,
+      "learning_rate": 3.132911392405064e-05,
+      "loss": 0.7254366278648376,
+      "step": 99,
+      "token_acc": 0.8042974184572577
+    },
+    {
+      "epoch": 0.25388765471278957,
+      "grad_norm": 4.740070832115608,
+      "learning_rate": 3.1645569620253167e-05,
+      "loss": 0.71342533826828,
+      "step": 100,
+      "token_acc": 0.8091580057237536
+    },
+    {
+      "epoch": 0.2564265312599175,
+      "grad_norm": 5.296776098340419,
+      "learning_rate": 3.1962025316455695e-05,
+      "loss": 0.7370377779006958,
+      "step": 101,
+      "token_acc": 0.8084075636818253
+    },
+    {
+      "epoch": 0.25896540780704536,
+      "grad_norm": 4.396582033002235,
+      "learning_rate": 3.227848101265823e-05,
+      "loss": 0.671318531036377,
+      "step": 102,
+      "token_acc": 0.8183896782622897
+    },
+    {
+      "epoch": 0.2615042843541733,
+      "grad_norm": 2.6335454183216696,
+      "learning_rate": 3.2594936708860766e-05,
+      "loss": 0.6583906412124634,
+      "step": 103,
+      "token_acc": 0.8211532014750251
+    },
+    {
+      "epoch": 0.2640431609013012,
+      "grad_norm": 6.236411278449773,
+      "learning_rate": 3.291139240506329e-05,
+      "loss": 0.7536572217941284,
+      "step": 104,
+      "token_acc": 0.7975090397750101
+    },
+    {
+      "epoch": 0.26658203744842907,
+      "grad_norm": 2.5341922649732602,
+      "learning_rate": 3.322784810126582e-05,
+      "loss": 0.6701930165290833,
+      "step": 105,
+      "token_acc": 0.8116928446771379
+    },
+    {
+      "epoch": 0.269120913995557,
+      "grad_norm": 11.881079085435806,
+      "learning_rate": 3.354430379746836e-05,
+      "loss": 0.5909067392349243,
+      "step": 106,
+      "token_acc": 0.8345961631468645
+    },
+    {
+      "epoch": 0.27165979054268485,
+      "grad_norm": 2.384697440414924,
+      "learning_rate": 3.386075949367089e-05,
+      "loss": 0.6282612085342407,
+      "step": 107,
+      "token_acc": 0.823628821631587
+    },
+    {
+      "epoch": 0.2741986670898128,
+      "grad_norm": 2.1995079597886673,
+      "learning_rate": 3.4177215189873416e-05,
+      "loss": 0.6936606764793396,
+      "step": 108,
+      "token_acc": 0.8020663404023926
+    },
+    {
+      "epoch": 0.27673754363694064,
+      "grad_norm": 4.861432353099602,
+      "learning_rate": 3.449367088607595e-05,
+      "loss": 0.778184175491333,
+      "step": 109,
+      "token_acc": 0.78875
+    },
+    {
+      "epoch": 0.27927642018406856,
+      "grad_norm": 9.57103993272581,
+      "learning_rate": 3.4810126582278487e-05,
+      "loss": 0.6751368641853333,
+      "step": 110,
+      "token_acc": 0.8148423557406306
+    },
+    {
+      "epoch": 0.28181529673119643,
+      "grad_norm": 13.549564951896668,
+      "learning_rate": 3.5126582278481015e-05,
+      "loss": 0.7030066251754761,
+      "step": 111,
+      "token_acc": 0.805439330543933
+    },
+    {
+      "epoch": 0.28435417327832435,
+      "grad_norm": 5.1073725117939475,
+      "learning_rate": 3.5443037974683544e-05,
+      "loss": 0.6886919736862183,
+      "step": 112,
+      "token_acc": 0.8083333333333333
+    },
+    {
+      "epoch": 0.2868930498254522,
+      "grad_norm": 22.063583297317187,
+      "learning_rate": 3.575949367088608e-05,
+      "loss": 0.6610169410705566,
+      "step": 113,
+      "token_acc": 0.8150403409955853
+    },
+    {
+      "epoch": 0.28943192637258014,
+      "grad_norm": 18.72026765015026,
+      "learning_rate": 3.607594936708861e-05,
+      "loss": 0.6841698884963989,
+      "step": 114,
+      "token_acc": 0.8123032904148784
+    },
+    {
+      "epoch": 0.291970802919708,
+      "grad_norm": 2.864881799429328,
+      "learning_rate": 3.639240506329114e-05,
+      "loss": 0.6466969847679138,
+      "step": 115,
+      "token_acc": 0.8172187406744256
+    },
+    {
+      "epoch": 0.2945096794668359,
+      "grad_norm": 6.716337706090099,
+      "learning_rate": 3.670886075949367e-05,
+      "loss": 0.6078028678894043,
+      "step": 116,
+      "token_acc": 0.8346497252747253
+    },
+    {
+      "epoch": 0.2970485560139638,
+      "grad_norm": 5.564745491758841,
+      "learning_rate": 3.70253164556962e-05,
+      "loss": 0.664849042892456,
+      "step": 117,
+      "token_acc": 0.8190188679245283
+    },
+    {
+      "epoch": 0.2995874325610917,
+      "grad_norm": 28.630717891823956,
+      "learning_rate": 3.7341772151898736e-05,
+      "loss": 0.6384534239768982,
+      "step": 118,
+      "token_acc": 0.8220830070477682
+    },
+    {
+      "epoch": 0.30212630910821964,
+      "grad_norm": 12.273392356907912,
+      "learning_rate": 3.765822784810127e-05,
+      "loss": 0.6201878190040588,
+      "step": 119,
+      "token_acc": 0.8250733897427042
+    },
+    {
+      "epoch": 0.3046651856553475,
+      "grad_norm": 4.192852448588315,
+      "learning_rate": 3.79746835443038e-05,
+      "loss": 0.6979496479034424,
+      "step": 120,
+      "token_acc": 0.8104520945665699
+    },
+    {
+      "epoch": 0.3072040622024754,
+      "grad_norm": 3.4657196050906567,
+      "learning_rate": 3.829113924050633e-05,
+      "loss": 0.6550031304359436,
+      "step": 121,
+      "token_acc": 0.8187575798409917
+    },
+    {
+      "epoch": 0.3097429387496033,
+      "grad_norm": 27.787374024365484,
+      "learning_rate": 3.8607594936708864e-05,
+      "loss": 0.6515278220176697,
+      "step": 122,
+      "token_acc": 0.8171852329517976
+    },
+    {
+      "epoch": 0.3122818152967312,
+      "grad_norm": 4.0417548472450715,
+      "learning_rate": 3.89240506329114e-05,
+      "loss": 0.605487585067749,
+      "step": 123,
+      "token_acc": 0.8265876933201711
+    },
+    {
+      "epoch": 0.3148206918438591,
+      "grad_norm": 3.8292748886400765,
+      "learning_rate": 3.924050632911392e-05,
+      "loss": 0.6631138324737549,
+      "step": 124,
+      "token_acc": 0.8129932356257046
+    },
+    {
+      "epoch": 0.317359568390987,
+      "grad_norm": 8.960479693493973,
+      "learning_rate": 3.9556962025316456e-05,
+      "loss": 0.6087026596069336,
+      "step": 125,
+      "token_acc": 0.8281978055012776
+    },
+    {
+      "epoch": 0.31989844493811487,
+      "grad_norm": 4.011722457924042,
+      "learning_rate": 3.987341772151899e-05,
+      "loss": 0.7122887372970581,
+      "step": 126,
+      "token_acc": 0.8093482748451785
+    },
+    {
+      "epoch": 0.3224373214852428,
+      "grad_norm": 31.39830164995725,
+      "learning_rate": 4.018987341772152e-05,
+      "loss": 0.6663644313812256,
+      "step": 127,
+      "token_acc": 0.8180548501777553
+    },
+    {
+      "epoch": 0.32497619803237066,
+      "grad_norm": 4.425596331791689,
+      "learning_rate": 4.050632911392405e-05,
+      "loss": 0.6300691366195679,
+      "step": 128,
+      "token_acc": 0.8203907404367099
+    },
+    {
+      "epoch": 0.3275150745794986,
+      "grad_norm": 16.1083849209539,
+      "learning_rate": 4.0822784810126584e-05,
+      "loss": 0.6704287528991699,
+      "step": 129,
+      "token_acc": 0.8078627591136526
+    },
+    {
+      "epoch": 0.33005395112662644,
+      "grad_norm": 2.2389483956009752,
+      "learning_rate": 4.113924050632912e-05,
+      "loss": 0.7170148491859436,
+      "step": 130,
+      "token_acc": 0.8084219603334292
+    },
+    {
+      "epoch": 0.33259282767375437,
+      "grad_norm": 3.613513118589346,
+      "learning_rate": 4.145569620253165e-05,
+      "loss": 0.6717650890350342,
+      "step": 131,
+      "token_acc": 0.8138652912621359
+    },
+    {
+      "epoch": 0.3351317042208823,
+      "grad_norm": 5.5827028571809105,
+      "learning_rate": 4.177215189873418e-05,
+      "loss": 0.6085385084152222,
+      "step": 132,
+      "token_acc": 0.8274016203703703
+    },
+    {
+      "epoch": 0.33767058076801015,
+      "grad_norm": 5.37295628319291,
+      "learning_rate": 4.208860759493671e-05,
+      "loss": 0.6445047855377197,
+      "step": 133,
+      "token_acc": 0.8134654130288784
+    },
+    {
+      "epoch": 0.3402094573151381,
+      "grad_norm": 2.1788757767041003,
+      "learning_rate": 4.240506329113924e-05,
+      "loss": 0.5425457954406738,
+      "step": 134,
+      "token_acc": 0.837465564738292
+    },
+    {
+      "epoch": 0.34274833386226594,
+      "grad_norm": 4.649401574423171,
+      "learning_rate": 4.2721518987341776e-05,
+      "loss": 0.5962692499160767,
+      "step": 135,
+      "token_acc": 0.8225226767071709
+    },
+    {
+      "epoch": 0.34528721040939386,
+      "grad_norm": 3.1742303151683506,
+      "learning_rate": 4.3037974683544305e-05,
+      "loss": 0.6038417816162109,
+      "step": 136,
+      "token_acc": 0.8316648698598491
+    },
+    {
+      "epoch": 0.34782608695652173,
+      "grad_norm": 5.696983011175984,
+      "learning_rate": 4.3354430379746834e-05,
+      "loss": 0.6016777753829956,
+      "step": 137,
+      "token_acc": 0.8321018707750354
+    },
+    {
+      "epoch": 0.35036496350364965,
+      "grad_norm": 5.797388182589766,
+      "learning_rate": 4.367088607594937e-05,
+      "loss": 0.6565921306610107,
+      "step": 138,
+      "token_acc": 0.8184429761562769
+    },
+    {
+      "epoch": 0.3529038400507775,
+      "grad_norm": 6.854274250412749,
+      "learning_rate": 4.3987341772151904e-05,
+      "loss": 0.691247820854187,
+      "step": 139,
+      "token_acc": 0.8143437786810017
+    },
+    {
+      "epoch": 0.35544271659790544,
+      "grad_norm": 2.6184869642829267,
+      "learning_rate": 4.430379746835443e-05,
+      "loss": 0.6285054683685303,
+      "step": 140,
+      "token_acc": 0.8222631355332106
+    },
+    {
+      "epoch": 0.3579815931450333,
+      "grad_norm": 4.653346904540878,
+      "learning_rate": 4.462025316455696e-05,
+      "loss": 0.5671603679656982,
+      "step": 141,
+      "token_acc": 0.8380648412565094
+    },
+    {
+      "epoch": 0.3605204696921612,
+      "grad_norm": 4.522681266549905,
+      "learning_rate": 4.49367088607595e-05,
+      "loss": 0.6068707704544067,
+      "step": 142,
+      "token_acc": 0.8331301432490095
+    },
+    {
+      "epoch": 0.3630593462392891,
+      "grad_norm": 3.181075099904124,
+      "learning_rate": 4.525316455696203e-05,
+      "loss": 0.5732223987579346,
+      "step": 143,
+      "token_acc": 0.836748182419035
+    },
+    {
+      "epoch": 0.365598222786417,
+      "grad_norm": 5.248411142229925,
+      "learning_rate": 4.556962025316456e-05,
+      "loss": 0.7090050578117371,
+      "step": 144,
+      "token_acc": 0.8088770652478298
+    },
+    {
+      "epoch": 0.3681370993335449,
+      "grad_norm": 2.2841820115958877,
+      "learning_rate": 4.588607594936709e-05,
+      "loss": 0.6395794153213501,
+      "step": 145,
+      "token_acc": 0.8251766217084137
+    },
+    {
+      "epoch": 0.3706759758806728,
+      "grad_norm": 3.192951403479268,
+      "learning_rate": 4.6202531645569625e-05,
+      "loss": 0.5619275569915771,
+      "step": 146,
+      "token_acc": 0.8353532367720957
+    },
+    {
+      "epoch": 0.3732148524278007,
+      "grad_norm": 6.184982774052421,
+      "learning_rate": 4.6518987341772154e-05,
+      "loss": 0.6076276302337646,
+      "step": 147,
+      "token_acc": 0.8321759259259259
+    },
+    {
+      "epoch": 0.3757537289749286,
+      "grad_norm": 3.9260670039543686,
+      "learning_rate": 4.683544303797468e-05,
+      "loss": 0.5210973024368286,
+      "step": 148,
+      "token_acc": 0.8457633053221288
+    },
+    {
+      "epoch": 0.3782926055220565,
+      "grad_norm": 3.6953261186445467,
+      "learning_rate": 4.715189873417722e-05,
+      "loss": 0.4648306369781494,
+      "step": 149,
+      "token_acc": 0.8572195383789587
+    },
+    {
+      "epoch": 0.3808314820691844,
+      "grad_norm": 4.9921203101786045,
+      "learning_rate": 4.7468354430379746e-05,
+      "loss": 0.676045835018158,
+      "step": 150,
+      "token_acc": 0.8171926006528836
+    },
+    {
+      "epoch": 0.3833703586163123,
+      "grad_norm": 20.42938551967448,
+      "learning_rate": 4.778481012658228e-05,
+      "loss": 0.619693398475647,
+      "step": 151,
+      "token_acc": 0.8338338338338338
+    },
+    {
+      "epoch": 0.38590923516344017,
+      "grad_norm": 8.755403481608779,
+      "learning_rate": 4.810126582278481e-05,
+      "loss": 0.5743368864059448,
+      "step": 152,
+      "token_acc": 0.8412815319462346
+    },
+    {
+      "epoch": 0.3884481117105681,
+      "grad_norm": 3.2072760680708425,
+      "learning_rate": 4.8417721518987346e-05,
+      "loss": 0.6492601633071899,
+      "step": 153,
+      "token_acc": 0.8137201735357917
+    },
+    {
+      "epoch": 0.39098698825769596,
+      "grad_norm": 3.8316550813580914,
+      "learning_rate": 4.8734177215189874e-05,
+      "loss": 0.6361663341522217,
+      "step": 154,
+      "token_acc": 0.8231698225734307
+    },
+    {
+      "epoch": 0.3935258648048239,
+      "grad_norm": 3.972772394188529,
+      "learning_rate": 4.905063291139241e-05,
+      "loss": 0.6939312815666199,
+      "step": 155,
+      "token_acc": 0.8113469274517964
+    },
+    {
+      "epoch": 0.39606474135195174,
+      "grad_norm": 16.657647846984553,
+      "learning_rate": 4.936708860759494e-05,
+      "loss": 0.6008589863777161,
+      "step": 156,
+      "token_acc": 0.8316379183252248
+    },
+    {
+      "epoch": 0.39860361789907967,
+      "grad_norm": 2.814742590512491,
+      "learning_rate": 4.968354430379747e-05,
+      "loss": 0.5716952085494995,
+      "step": 157,
+      "token_acc": 0.8305734227453128
+    },
+    {
+      "epoch": 0.40114249444620753,
+      "grad_norm": 3.3183075108254605,
+      "learning_rate": 5e-05,
+      "loss": 0.5892596244812012,
+      "step": 158,
+      "token_acc": 0.823594674556213
+    },
+    {
+      "epoch": 0.40368137099333545,
+      "grad_norm": 3.805622229512587,
+      "learning_rate": 4.9999689166542295e-05,
+      "loss": 0.5727576017379761,
+      "step": 159,
+      "token_acc": 0.832796486090776
+    },
+    {
+      "epoch": 0.4062202475404633,
+      "grad_norm": 2.9959092236114975,
+      "learning_rate": 4.999875667389858e-05,
+      "loss": 0.5001785159111023,
+      "step": 160,
+      "token_acc": 0.8474907244678773
+    },
+    {
+      "epoch": 0.40875912408759124,
+      "grad_norm": 2.926865350121703,
+      "learning_rate": 4.999720254525684e-05,
+      "loss": 0.6166025400161743,
+      "step": 161,
+      "token_acc": 0.820961340585036
+    },
+    {
+      "epoch": 0.41129800063471916,
+      "grad_norm": 2.2665405249006403,
+      "learning_rate": 4.999502681926309e-05,
+      "loss": 0.5835655927658081,
+      "step": 162,
+      "token_acc": 0.8314150304671631
+    },
+    {
+      "epoch": 0.41383687718184703,
+      "grad_norm": 6.077225672698818,
+      "learning_rate": 4.999222955002041e-05,
+      "loss": 0.6306543350219727,
+      "step": 163,
+      "token_acc": 0.8202965582913889
+    },
+    {
+      "epoch": 0.41637575372897495,
+      "grad_norm": 9.205523914845546,
+      "learning_rate": 4.9988810807087584e-05,
+      "loss": 0.6070675253868103,
+      "step": 164,
+      "token_acc": 0.8314500941619586
+    },
+    {
+      "epoch": 0.4189146302761028,
+      "grad_norm": 2.520164837218412,
+      "learning_rate": 4.99847706754774e-05,
+      "loss": 0.5549716949462891,
+      "step": 165,
+      "token_acc": 0.8379689521345407
+    },
+    {
+      "epoch": 0.42145350682323074,
+      "grad_norm": 2.3723529006989215,
+      "learning_rate": 4.998010925565448e-05,
+      "loss": 0.5140861868858337,
+      "step": 166,
+      "token_acc": 0.8497000856898029
+    },
+    {
+      "epoch": 0.4239923833703586,
+      "grad_norm": 3.168399232927889,
+      "learning_rate": 4.997482666353287e-05,
+      "loss": 0.5713208913803101,
+      "step": 167,
+      "token_acc": 0.8342638379326847
+    },
+    {
+      "epoch": 0.4265312599174865,
+      "grad_norm": 3.1544527876057353,
+      "learning_rate": 4.996892303047306e-05,
+      "loss": 0.5652569532394409,
+      "step": 168,
+      "token_acc": 0.8433680398370303
+    },
+    {
+      "epoch": 0.4290701364646144,
+      "grad_norm": 4.050502290302643,
+      "learning_rate": 4.99623985032788e-05,
+      "loss": 0.6010338068008423,
+      "step": 169,
+      "token_acc": 0.8299908842297175
+    },
+    {
+      "epoch": 0.4316090130117423,
+      "grad_norm": 2.3208744483079924,
+      "learning_rate": 4.9955253244193375e-05,
+      "loss": 0.625551700592041,
+      "step": 170,
+      "token_acc": 0.8214095744680852
+    },
+    {
+      "epoch": 0.4341478895588702,
+      "grad_norm": 3.5752910543983734,
+      "learning_rate": 4.994748743089566e-05,
+      "loss": 0.5763528347015381,
+      "step": 171,
+      "token_acc": 0.8370288248337029
+    },
+    {
+      "epoch": 0.4366867661059981,
+      "grad_norm": 5.186367702439989,
+      "learning_rate": 4.993910125649561e-05,
+      "loss": 0.583060085773468,
+      "step": 172,
+      "token_acc": 0.8326601269474899
+    },
+    {
+      "epoch": 0.43922564265312597,
+      "grad_norm": 2.314721989016231,
+      "learning_rate": 4.9930094929529506e-05,
+      "loss": 0.6205261945724487,
+      "step": 173,
+      "token_acc": 0.8203899721448468
+    },
+    {
+      "epoch": 0.4417645192002539,
+      "grad_norm": 3.3583225237625016,
+      "learning_rate": 4.992046867395478e-05,
+      "loss": 0.5230633020401001,
+      "step": 174,
+      "token_acc": 0.8419920582395765
+    },
+    {
+      "epoch": 0.44430339574738176,
+      "grad_norm": 4.042306283685085,
+      "learning_rate": 4.99102227291444e-05,
+      "loss": 0.5143859386444092,
+      "step": 175,
+      "token_acc": 0.8474544778892605
+    },
+    {
+      "epoch": 0.4468422722945097,
+      "grad_norm": 2.6321524863816603,
+      "learning_rate": 4.989935734988098e-05,
+      "loss": 0.6144051551818848,
+      "step": 176,
+      "token_acc": 0.8175049636002647
+    },
+    {
+      "epoch": 0.4493811488416376,
+      "grad_norm": 2.361660215036252,
+      "learning_rate": 4.988787280635038e-05,
+      "loss": 0.49258965253829956,
+      "step": 177,
+      "token_acc": 0.8459661430532256
+    },
+    {
+      "epoch": 0.45192002538876547,
+      "grad_norm": 2.0369998726790266,
+      "learning_rate": 4.987576938413504e-05,
+      "loss": 0.5039229393005371,
+      "step": 178,
+      "token_acc": 0.8487822609960014
+    },
+    {
+      "epoch": 0.4544589019358934,
+      "grad_norm": 6.93229029448243,
+      "learning_rate": 4.9863047384206835e-05,
+      "loss": 0.5814566612243652,
+      "step": 179,
+      "token_acc": 0.8274802458296752
+    },
+    {
+      "epoch": 0.45699777848302126,
+      "grad_norm": 1.9442137330470775,
+      "learning_rate": 4.984970712291963e-05,
+      "loss": 0.5427694916725159,
+      "step": 180,
+      "token_acc": 0.8449736295349208
+    },
+    {
+      "epoch": 0.4595366550301492,
+      "grad_norm": 3.8278365677194706,
+      "learning_rate": 4.983574893200139e-05,
+      "loss": 0.5021108388900757,
+      "step": 181,
+      "token_acc": 0.8494821249582358
+    },
+    {
+      "epoch": 0.46207553157727704,
+      "grad_norm": 3.898165087457517,
+      "learning_rate": 4.9821173158545936e-05,
+      "loss": 0.529310941696167,
+      "step": 182,
+      "token_acc": 0.8456512269493248
+    },
+    {
+      "epoch": 0.46461440812440497,
+      "grad_norm": 5.784138022488126,
+      "learning_rate": 4.9805980165004304e-05,
+      "loss": 0.5193842053413391,
+      "step": 183,
+      "token_acc": 0.8486529318541997
+    },
+    {
+      "epoch": 0.46715328467153283,
+      "grad_norm": 2.7860896602561755,
+      "learning_rate": 4.9790170329175754e-05,
+      "loss": 0.5510420799255371,
+      "step": 184,
+      "token_acc": 0.8427399903521466
+    },
+    {
+      "epoch": 0.46969216121866075,
+      "grad_norm": 2.391478502605225,
+      "learning_rate": 4.977374404419837e-05,
+      "loss": 0.565636396408081,
+      "step": 185,
+      "token_acc": 0.8384972889233152
+    },
+    {
+      "epoch": 0.4722310377657886,
+      "grad_norm": 2.7381469955000233,
+      "learning_rate": 4.975670171853926e-05,
+      "loss": 0.5837178230285645,
+      "step": 186,
+      "token_acc": 0.8304386223138063
+    },
+    {
+      "epoch": 0.47476991431291654,
+      "grad_norm": 4.436221537482703,
+      "learning_rate": 4.973904377598443e-05,
+      "loss": 0.5879358649253845,
+      "step": 187,
+      "token_acc": 0.8254743303571429
+    },
+    {
+      "epoch": 0.4773087908600444,
+      "grad_norm": 3.161812435220317,
+      "learning_rate": 4.972077065562821e-05,
+      "loss": 0.5809712409973145,
+      "step": 188,
+      "token_acc": 0.8361007729745205
+    },
+    {
+      "epoch": 0.47984766740717233,
+      "grad_norm": 1.5520306180347663,
+      "learning_rate": 4.970188281186241e-05,
+      "loss": 0.5579955577850342,
+      "step": 189,
+      "token_acc": 0.837281009110021
+    },
+    {
+      "epoch": 0.4823865439543002,
+      "grad_norm": 3.727248476475511,
+      "learning_rate": 4.9682380714364897e-05,
+      "loss": 0.58579421043396,
+      "step": 190,
+      "token_acc": 0.8350501304765829
+    },
+    {
+      "epoch": 0.4849254205014281,
+      "grad_norm": 3.0795046455062614,
+      "learning_rate": 4.9662264848088034e-05,
+      "loss": 0.4905577301979065,
+      "step": 191,
+      "token_acc": 0.8543485289363077
+    },
+    {
+      "epoch": 0.48746429704855604,
+      "grad_norm": 2.222320433523395,
+      "learning_rate": 4.964153571324658e-05,
+      "loss": 0.6000843048095703,
+      "step": 192,
+      "token_acc": 0.8202116862403944
+    },
+    {
+      "epoch": 0.4900031735956839,
+      "grad_norm": 3.054493416036562,
+      "learning_rate": 4.962019382530521e-05,
+      "loss": 0.553142786026001,
+      "step": 193,
+      "token_acc": 0.8378803028138837
+    },
+    {
+      "epoch": 0.4925420501428118,
+      "grad_norm": 1.48942863662551,
+      "learning_rate": 4.959823971496574e-05,
+      "loss": 0.49760788679122925,
+      "step": 194,
+      "token_acc": 0.848271341962969
+    },
+    {
+      "epoch": 0.4950809266899397,
+      "grad_norm": 1.4337417758608024,
+      "learning_rate": 4.9575673928153957e-05,
+      "loss": 0.5126315355300903,
+      "step": 195,
+      "token_acc": 0.846869578930768
+    },
+    {
+      "epoch": 0.4976198032370676,
+      "grad_norm": 1.8216377101814463,
+      "learning_rate": 4.9552497026005974e-05,
+      "loss": 0.5140909552574158,
+      "step": 196,
+      "token_acc": 0.8483076923076923
+    },
+    {
+      "epoch": 0.5001586797841955,
+      "grad_norm": 3.6989217579732125,
+      "learning_rate": 4.952870958485432e-05,
+      "loss": 0.5483355522155762,
+      "step": 197,
+      "token_acc": 0.8436149991359945
+    },
+    {
+      "epoch": 0.5026975563313234,
+      "grad_norm": 1.4793601703477388,
+      "learning_rate": 4.9504312196213596e-05,
+      "loss": 0.5258674621582031,
+      "step": 198,
+      "token_acc": 0.844776119402985
+    },
+    {
+      "epoch": 0.5052364328784513,
+      "grad_norm": 3.320358649604752,
+      "learning_rate": 4.947930546676579e-05,
+      "loss": 0.5483481287956238,
+      "step": 199,
+      "token_acc": 0.8407105847520355
+    },
+    {
+      "epoch": 0.5077753094255791,
+      "grad_norm": 1.4836965410501335,
+      "learning_rate": 4.9453690018345144e-05,
+      "loss": 0.5698910355567932,
+      "step": 200,
+      "token_acc": 0.8355416991426344
+    },
+    {
+      "epoch": 0.5103141859727071,
+      "grad_norm": 1.9840806554207313,
+      "learning_rate": 4.942746648792274e-05,
+      "loss": 0.6226257681846619,
+      "step": 201,
+      "token_acc": 0.821941594317285
+    },
+    {
+      "epoch": 0.512853062519835,
+      "grad_norm": 3.4200124483670358,
+      "learning_rate": 4.940063552759061e-05,
+      "loss": 0.5485865473747253,
+      "step": 202,
+      "token_acc": 0.8423786494716102
+    },
+    {
+      "epoch": 0.5153919390669629,
+      "grad_norm": 1.4137645693467988,
+      "learning_rate": 4.937319780454559e-05,
+      "loss": 0.4833056330680847,
+      "step": 203,
+      "token_acc": 0.8569816188326347
+    },
+    {
+      "epoch": 0.5179308156140907,
+      "grad_norm": 2.216132222614037,
+      "learning_rate": 4.934515400107266e-05,
+      "loss": 0.4503518342971802,
+      "step": 204,
+      "token_acc": 0.8646072954321393
+    },
+    {
+      "epoch": 0.5204696921612186,
+      "grad_norm": 4.426951079644797,
+      "learning_rate": 4.931650481452801e-05,
+      "loss": 0.5067535638809204,
+      "step": 205,
+      "token_acc": 0.8500777604976671
+    },
+    {
+      "epoch": 0.5230085687083466,
+      "grad_norm": 7.154300916510774,
+      "learning_rate": 4.928725095732169e-05,
+      "loss": 0.5493313670158386,
+      "step": 206,
+      "token_acc": 0.8378132118451025
+    },
+    {
+      "epoch": 0.5255474452554745,
+      "grad_norm": 1.965712161044352,
+      "learning_rate": 4.925739315689991e-05,
+      "loss": 0.4918142557144165,
+      "step": 207,
+      "token_acc": 0.8530601240226476
+    },
+    {
+      "epoch": 0.5280863218026024,
+      "grad_norm": 1.5124863710410141,
+      "learning_rate": 4.922693215572695e-05,
+      "loss": 0.5023689270019531,
+      "step": 208,
+      "token_acc": 0.8512195121951219
+    },
+    {
+      "epoch": 0.5306251983497302,
+      "grad_norm": 2.3375825804294723,
+      "learning_rate": 4.919586871126667e-05,
+      "loss": 0.5670746564865112,
+      "step": 209,
+      "token_acc": 0.8375552282768778
+    },
+    {
+      "epoch": 0.5331640748968581,
+      "grad_norm": 1.8418248968647637,
+      "learning_rate": 4.916420359596368e-05,
+      "loss": 0.5001412630081177,
+      "step": 210,
+      "token_acc": 0.8473023413640991
+    },
+    {
+      "epoch": 0.535702951443986,
+      "grad_norm": 2.4235777237942924,
+      "learning_rate": 4.9131937597224185e-05,
+      "loss": 0.5839577913284302,
+      "step": 211,
+      "token_acc": 0.8330151270377442
+    },
+    {
+      "epoch": 0.538241827991114,
+      "grad_norm": 15.841289330503669,
+      "learning_rate": 4.909907151739633e-05,
+      "loss": 0.5890910625457764,
+      "step": 212,
+      "token_acc": 0.8299887822510283
+    },
+    {
+      "epoch": 0.5407807045382418,
+      "grad_norm": 2.1298845605680197,
+      "learning_rate": 4.90656061737503e-05,
+      "loss": 0.5155512094497681,
+      "step": 213,
+      "token_acc": 0.8465608465608465
+    },
+    {
+      "epoch": 0.5433195810853697,
+      "grad_norm": 2.1579493985693454,
+      "learning_rate": 4.9031542398457974e-05,
+      "loss": 0.5869525671005249,
+      "step": 214,
+      "token_acc": 0.8264487369985141
+    },
+    {
+      "epoch": 0.5458584576324976,
+      "grad_norm": 3.015091026973871,
+      "learning_rate": 4.899688103857223e-05,
+      "loss": 0.5348740816116333,
+      "step": 215,
+      "token_acc": 0.8366959820658735
+    },
+    {
+      "epoch": 0.5483973341796256,
+      "grad_norm": 1.4983193386255376,
+      "learning_rate": 4.896162295600589e-05,
+      "loss": 0.5215170383453369,
+      "step": 216,
+      "token_acc": 0.8455044322498945
+    },
+    {
+      "epoch": 0.5509362107267534,
+      "grad_norm": 7.6369461867236215,
+      "learning_rate": 4.892576902751031e-05,
+      "loss": 0.6286407709121704,
+      "step": 217,
+      "token_acc": 0.8244441744623983
+    },
+    {
+      "epoch": 0.5534750872738813,
+      "grad_norm": 1.909191100547535,
+      "learning_rate": 4.888932014465352e-05,
+      "loss": 0.5405088663101196,
+      "step": 218,
+      "token_acc": 0.8398148148148148
+    },
+    {
+      "epoch": 0.5560139638210092,
+      "grad_norm": 2.1249944661063713,
+      "learning_rate": 4.8852277213798106e-05,
+      "loss": 0.5207107067108154,
+      "step": 219,
+      "token_acc": 0.8505783057205377
+    },
+    {
+      "epoch": 0.5585528403681371,
+      "grad_norm": 4.590133511444357,
+      "learning_rate": 4.881464115607865e-05,
+      "loss": 0.5813536643981934,
+      "step": 220,
+      "token_acc": 0.831871745683749
+    },
+    {
+      "epoch": 0.561091716915265,
+      "grad_norm": 1.641052016603602,
+      "learning_rate": 4.877641290737884e-05,
+      "loss": 0.5410237312316895,
+      "step": 221,
+      "token_acc": 0.8366046831955923
+    },
+    {
+      "epoch": 0.5636305934623929,
+      "grad_norm": 2.277495246079136,
+      "learning_rate": 4.8737593418308156e-05,
+      "loss": 0.5533527135848999,
+      "step": 222,
+      "token_acc": 0.8363823756763721
+    },
+    {
+      "epoch": 0.5661694700095208,
+      "grad_norm": 4.159566523283519,
+      "learning_rate": 4.86981836541783e-05,
+      "loss": 0.559136152267456,
+      "step": 223,
+      "token_acc": 0.8388266834884087
+    },
+    {
+      "epoch": 0.5687083465566487,
+      "grad_norm": 225.83355044511626,
+      "learning_rate": 4.865818459497911e-05,
+      "loss": 0.4840168356895447,
+      "step": 224,
+      "token_acc": 0.8560429982568274
+    },
+    {
+      "epoch": 0.5712472231037766,
+      "grad_norm": 1.7941761886315637,
+      "learning_rate": 4.861759723535426e-05,
+      "loss": 0.5671250224113464,
+      "step": 225,
+      "token_acc": 0.8405020448455789
+    },
+    {
+      "epoch": 0.5737860996509044,
+      "grad_norm": 1.6927695579956952,
+      "learning_rate": 4.8576422584576514e-05,
+      "loss": 0.5532321929931641,
+      "step": 226,
+      "token_acc": 0.8365077158348394
+    },
+    {
+      "epoch": 0.5763249761980324,
+      "grad_norm": 1.3677369838188163,
+      "learning_rate": 4.8534661666522584e-05,
+      "loss": 0.5039372444152832,
+      "step": 227,
+      "token_acc": 0.8533067362283259
+    },
+    {
+      "epoch": 0.5788638527451603,
+      "grad_norm": 3.1985990377060944,
+      "learning_rate": 4.849231551964771e-05,
+      "loss": 0.4986110031604767,
+      "step": 228,
+      "token_acc": 0.8453477868112015
+    },
+    {
+      "epoch": 0.5814027292922882,
+      "grad_norm": 2.5449576622123646,
+      "learning_rate": 4.844938519695984e-05,
+      "loss": 0.5104832649230957,
+      "step": 229,
+      "token_acc": 0.8486502433987314
+    },
+    {
+      "epoch": 0.583941605839416,
+      "grad_norm": 1.3118749339834277,
+      "learning_rate": 4.8405871765993433e-05,
+      "loss": 0.505854606628418,
+      "step": 230,
+      "token_acc": 0.8532082324455206
+    },
+    {
+      "epoch": 0.5864804823865439,
+      "grad_norm": 2.18900446561014,
+      "learning_rate": 4.836177630878289e-05,
+      "loss": 0.5590152144432068,
+      "step": 231,
+      "token_acc": 0.8363560267857143
+    },
+    {
+      "epoch": 0.5890193589336719,
+      "grad_norm": 15.30797015176157,
+      "learning_rate": 4.8317099921835697e-05,
+      "loss": 0.4845745265483856,
+      "step": 232,
+      "token_acc": 0.8525514089870525
+    },
+    {
+      "epoch": 0.5915582354807998,
+      "grad_norm": 1.7181163596035074,
+      "learning_rate": 4.827184371610511e-05,
+      "loss": 0.5797554850578308,
+      "step": 233,
+      "token_acc": 0.8373081140350878
+    },
+    {
+      "epoch": 0.5940971120279276,
+      "grad_norm": 1.1288420718760477,
+      "learning_rate": 4.822600881696256e-05,
+      "loss": 0.5012973546981812,
+      "step": 234,
+      "token_acc": 0.8529411764705882
+    },
+    {
+      "epoch": 0.5966359885750555,
+      "grad_norm": 1.437368423568747,
+      "learning_rate": 4.817959636416969e-05,
+      "loss": 0.5510119199752808,
+      "step": 235,
+      "token_acc": 0.8344022575679836
+    },
+    {
+      "epoch": 0.5991748651221834,
+      "grad_norm": 1.2890032605357924,
+      "learning_rate": 4.813260751184992e-05,
+      "loss": 0.5216597318649292,
+      "step": 236,
+      "token_acc": 0.8493268404468634
+    },
+    {
+      "epoch": 0.6017137416693114,
+      "grad_norm": 1.405196489873568,
+      "learning_rate": 4.808504342845986e-05,
+      "loss": 0.6565842032432556,
+      "step": 237,
+      "token_acc": 0.8143687707641196
+    },
+    {
+      "epoch": 0.6042526182164393,
+      "grad_norm": 1.6519014060456265,
+      "learning_rate": 4.803690529676019e-05,
+      "loss": 0.4384632706642151,
+      "step": 238,
+      "token_acc": 0.8647027694478744
+    },
+    {
+      "epoch": 0.6067914947635671,
+      "grad_norm": 1.8927813931053827,
+      "learning_rate": 4.7988194313786275e-05,
+      "loss": 0.504548192024231,
+      "step": 239,
+      "token_acc": 0.8548215641609719
+    },
+    {
+      "epoch": 0.609330371310695,
+      "grad_norm": 1.3750618958063838,
+      "learning_rate": 4.7938911690818347e-05,
+      "loss": 0.5044655799865723,
+      "step": 240,
+      "token_acc": 0.8456818530449456
+    },
+    {
+      "epoch": 0.6118692478578229,
+      "grad_norm": 1.7791425452801302,
+      "learning_rate": 4.7889058653351485e-05,
+      "loss": 0.4389927089214325,
+      "step": 241,
+      "token_acc": 0.8615710435117443
+    },
+    {
+      "epoch": 0.6144081244049509,
+      "grad_norm": 3.174913815120615,
+      "learning_rate": 4.783863644106502e-05,
+      "loss": 0.5367846488952637,
+      "step": 242,
+      "token_acc": 0.841294538665007
+    },
+    {
+      "epoch": 0.6169470009520787,
+      "grad_norm": 2.447734108467047,
+      "learning_rate": 4.778764630779183e-05,
+      "loss": 0.5005022287368774,
+      "step": 243,
+      "token_acc": 0.8491663435440093
+    },
+    {
+      "epoch": 0.6194858774992066,
+      "grad_norm": 2.807814103600755,
+      "learning_rate": 4.773608952148706e-05,
+      "loss": 0.5408118367195129,
+      "step": 244,
+      "token_acc": 0.8354211087420043
+    },
+    {
+      "epoch": 0.6220247540463345,
+      "grad_norm": 2.227992234463683,
+      "learning_rate": 4.7683967364196624e-05,
+      "loss": 0.5159645080566406,
+      "step": 245,
+      "token_acc": 0.8429188135357193
+    },
+    {
+      "epoch": 0.6245636305934624,
+      "grad_norm": 3.9276559319284314,
+      "learning_rate": 4.763128113202537e-05,
+      "loss": 0.5435395240783691,
+      "step": 246,
+      "token_acc": 0.8381928868952259
+    },
+    {
+      "epoch": 0.6271025071405902,
+      "grad_norm": 21.07592625562304,
+      "learning_rate": 4.7578032135104796e-05,
+      "loss": 0.528008222579956,
+      "step": 247,
+      "token_acc": 0.8437879893022125
+    },
+    {
+      "epoch": 0.6296413836877182,
+      "grad_norm": 2.4305528198591606,
+      "learning_rate": 4.752422169756048e-05,
+      "loss": 0.5454938411712646,
+      "step": 248,
+      "token_acc": 0.8377711162052548
+    },
+    {
+      "epoch": 0.6321802602348461,
+      "grad_norm": 4.774445766197441,
+      "learning_rate": 4.7469851157479177e-05,
+      "loss": 0.5896109342575073,
+      "step": 249,
+      "token_acc": 0.8303083960454478
+    },
+    {
+      "epoch": 0.634719136781974,
+      "grad_norm": 2.9285874898965036,
+      "learning_rate": 4.7414921866875524e-05,
+      "loss": 0.5258548259735107,
+      "step": 250,
+      "token_acc": 0.8459306342407075
+    },
+    {
+      "epoch": 0.6372580133291019,
+      "grad_norm": 1.1551732083505906,
+      "learning_rate": 4.7359435191658425e-05,
+      "loss": 0.4973874092102051,
+      "step": 251,
+      "token_acc": 0.8523009950248757
+    },
+    {
+      "epoch": 0.6397968898762297,
+      "grad_norm": 1.404758000704147,
+      "learning_rate": 4.730339251159709e-05,
+      "loss": 0.48979347944259644,
+      "step": 252,
+      "token_acc": 0.8519233625427002
+    },
+    {
+      "epoch": 0.6423357664233577,
+      "grad_norm": 1.1599030830108534,
+      "learning_rate": 4.724679522028672e-05,
+      "loss": 0.4671449661254883,
+      "step": 253,
+      "token_acc": 0.8547652417659425
+    },
+    {
+      "epoch": 0.6448746429704856,
+      "grad_norm": 9.767857767777004,
+      "learning_rate": 4.718964472511386e-05,
+      "loss": 0.5248907208442688,
+      "step": 254,
+      "token_acc": 0.8436238600687697
+    },
+    {
+      "epoch": 0.6474135195176135,
+      "grad_norm": 1.5836269105977552,
+      "learning_rate": 4.713194244722138e-05,
+      "loss": 0.492233008146286,
+      "step": 255,
+      "token_acc": 0.8537440902851914
+    },
+    {
+      "epoch": 0.6499523960647413,
+      "grad_norm": 1.676133573613313,
+      "learning_rate": 4.707368982147318e-05,
+      "loss": 0.49239009618759155,
+      "step": 256,
+      "token_acc": 0.8473175021987687
+    },
+    {
+      "epoch": 0.6524912726118692,
+      "grad_norm": 12.420578245961337,
+      "learning_rate": 4.701488829641845e-05,
+      "loss": 0.55488121509552,
+      "step": 257,
+      "token_acc": 0.835998984513836
+    },
+    {
+      "epoch": 0.6550301491589972,
+      "grad_norm": 2.7460132417830643,
+      "learning_rate": 4.6955539334255716e-05,
+      "loss": 0.5333112478256226,
+      "step": 258,
+      "token_acc": 0.8431862832648688
+    },
+    {
+      "epoch": 0.6575690257061251,
+      "grad_norm": 2.143768931475659,
+      "learning_rate": 4.6895644410796416e-05,
+      "loss": 0.5408412218093872,
+      "step": 259,
+      "token_acc": 0.8437373601186463
+    },
+    {
+      "epoch": 0.6601079022532529,
+      "grad_norm": 1.552814047782724,
+      "learning_rate": 4.6835205015428246e-05,
+      "loss": 0.5155550241470337,
+      "step": 260,
+      "token_acc": 0.8499210110584519
+    },
+    {
+      "epoch": 0.6626467788003808,
+      "grad_norm": 1.2034458590422716,
+      "learning_rate": 4.6774222651078106e-05,
+      "loss": 0.5099776983261108,
+      "step": 261,
+      "token_acc": 0.8472036789793799
+    },
+    {
+      "epoch": 0.6651856553475087,
+      "grad_norm": 2.924220695079595,
+      "learning_rate": 4.671269883417473e-05,
+      "loss": 0.5947083234786987,
+      "step": 262,
+      "token_acc": 0.8290450928381963
+    },
+    {
+      "epoch": 0.6677245318946367,
+      "grad_norm": 1.455955314620601,
+      "learning_rate": 4.665063509461097e-05,
+      "loss": 0.5068027973175049,
+      "step": 263,
+      "token_acc": 0.8482072342441952
+    },
+    {
+      "epoch": 0.6702634084417646,
+      "grad_norm": 3.344945263340371,
+      "learning_rate": 4.658803297570577e-05,
+      "loss": 0.5365261435508728,
+      "step": 264,
+      "token_acc": 0.8415192190359976
+    },
+    {
+      "epoch": 0.6728022849888924,
+      "grad_norm": 1.9005203964312098,
+      "learning_rate": 4.652489403416579e-05,
+      "loss": 0.4796498417854309,
+      "step": 265,
+      "token_acc": 0.8555090206185567
+    },
+    {
+      "epoch": 0.6753411615360203,
+      "grad_norm": 2.8726899682034412,
+      "learning_rate": 4.6461219840046654e-05,
+      "loss": 0.49148356914520264,
+      "step": 266,
+      "token_acc": 0.8587875255027688
+    },
+    {
+      "epoch": 0.6778800380831482,
+      "grad_norm": 3.620289535245779,
+      "learning_rate": 4.639701197671397e-05,
+      "loss": 0.5379594564437866,
+      "step": 267,
+      "token_acc": 0.844286636539703
+    },
+    {
+      "epoch": 0.6804189146302762,
+      "grad_norm": 1.880048630363598,
+      "learning_rate": 4.6332272040803895e-05,
+      "loss": 0.40906715393066406,
+      "step": 268,
+      "token_acc": 0.8677865920049774
+    },
+    {
+      "epoch": 0.682957791177404,
+      "grad_norm": 3.2529749263524854,
+      "learning_rate": 4.6267001642183496e-05,
+      "loss": 0.5698261857032776,
+      "step": 269,
+      "token_acc": 0.8295176385889129
+    },
+    {
+      "epoch": 0.6854966677245319,
+      "grad_norm": 1.4115756433255298,
+      "learning_rate": 4.620120240391065e-05,
+      "loss": 0.5403001308441162,
+      "step": 270,
+      "token_acc": 0.8392932267564156
+    },
+    {
+      "epoch": 0.6880355442716598,
+      "grad_norm": 1.5731929659655102,
+      "learning_rate": 4.613487596219376e-05,
+      "loss": 0.4380991756916046,
+      "step": 271,
+      "token_acc": 0.8581743166580712
+    },
+    {
+      "epoch": 0.6905744208187877,
+      "grad_norm": 2.429353942954586,
+      "learning_rate": 4.606802396635098e-05,
+      "loss": 0.4497607946395874,
+      "step": 272,
+      "token_acc": 0.8668831168831169
+    },
+    {
+      "epoch": 0.6931132973659155,
+      "grad_norm": 1.4740422171009888,
+      "learning_rate": 4.600064807876929e-05,
+      "loss": 0.4794745445251465,
+      "step": 273,
+      "token_acc": 0.8582345601996257
+    },
+    {
+      "epoch": 0.6956521739130435,
+      "grad_norm": 1.4962672531000716,
+      "learning_rate": 4.593274997486309e-05,
+      "loss": 0.5734583735466003,
+      "step": 274,
+      "token_acc": 0.8358989687640114
+    },
+    {
+      "epoch": 0.6981910504601714,
+      "grad_norm": 2.582140206521716,
+      "learning_rate": 4.586433134303257e-05,
+      "loss": 0.47569674253463745,
+      "step": 275,
+      "token_acc": 0.851392632524708
+    },
+    {
+      "epoch": 0.7007299270072993,
+      "grad_norm": 2.0299052997394145,
+      "learning_rate": 4.579539388462173e-05,
+      "loss": 0.4513978362083435,
+      "step": 276,
+      "token_acc": 0.8595333128645993
+    },
+    {
+      "epoch": 0.7032688035544271,
+      "grad_norm": 1.4683236839939144,
+      "learning_rate": 4.572593931387604e-05,
+      "loss": 0.4924212694168091,
+      "step": 277,
+      "token_acc": 0.8509630565203663
+    },
+    {
+      "epoch": 0.705807680101555,
+      "grad_norm": 1.6692647992426015,
+      "learning_rate": 4.5655969357899874e-05,
+      "loss": 0.50971919298172,
+      "step": 278,
+      "token_acc": 0.8504672897196262
+    },
+    {
+      "epoch": 0.708346556648683,
+      "grad_norm": 2.772573768796691,
+      "learning_rate": 4.5585485756613486e-05,
+      "loss": 0.4721433222293854,
+      "step": 279,
+      "token_acc": 0.8584571832979476
+    },
+    {
+      "epoch": 0.7108854331958109,
+      "grad_norm": 2.0060707686440407,
+      "learning_rate": 4.551449026270979e-05,
+      "loss": 0.5175821781158447,
+      "step": 280,
+      "token_acc": 0.8468809073724007
+    },
+    {
+      "epoch": 0.7134243097429388,
+      "grad_norm": 1.8487986104225613,
+      "learning_rate": 4.544298464161079e-05,
+      "loss": 0.5257778763771057,
+      "step": 281,
+      "token_acc": 0.8475199020208206
+    },
+    {
+      "epoch": 0.7159631862900666,
+      "grad_norm": 2.127782753744905,
+      "learning_rate": 4.537097067142363e-05,
+      "loss": 0.562999427318573,
+      "step": 282,
+      "token_acc": 0.8408823529411765
+    },
+    {
+      "epoch": 0.7185020628371945,
+      "grad_norm": 2.5685178275153957,
+      "learning_rate": 4.529845014289642e-05,
+      "loss": 0.5207107067108154,
+      "step": 283,
+      "token_acc": 0.8469147432878003
+    },
+    {
+      "epoch": 0.7210409393843225,
+      "grad_norm": 6.2775274916114014,
+      "learning_rate": 4.522542485937369e-05,
+      "loss": 0.5115345120429993,
+      "step": 284,
+      "token_acc": 0.8454268292682927
+    },
+    {
+      "epoch": 0.7235798159314504,
+      "grad_norm": 1.2979163949839545,
+      "learning_rate": 4.5151896636751556e-05,
+      "loss": 0.4864828586578369,
+      "step": 285,
+      "token_acc": 0.855954855954856
+    },
+    {
+      "epoch": 0.7261186924785782,
+      "grad_norm": 1.9952838067379783,
+      "learning_rate": 4.5077867303432546e-05,
+      "loss": 0.49415430426597595,
+      "step": 286,
+      "token_acc": 0.8480227882037533
+    },
+    {
+      "epoch": 0.7286575690257061,
+      "grad_norm": 1.7314556438703808,
+      "learning_rate": 4.500333870028016e-05,
+      "loss": 0.5226168036460876,
+      "step": 287,
+      "token_acc": 0.8493612242032851
+    },
+    {
+      "epoch": 0.731196445572834,
+      "grad_norm": 4.40006164102703,
+      "learning_rate": 4.4928312680573064e-05,
+      "loss": 0.4729849100112915,
+      "step": 288,
+      "token_acc": 0.8531379420197515
+    },
+    {
+      "epoch": 0.733735322119962,
+      "grad_norm": 6.394724509170586,
+      "learning_rate": 4.485279110995903e-05,
+      "loss": 0.5564934611320496,
+      "step": 289,
+      "token_acc": 0.8377503852080124
+    },
+    {
+      "epoch": 0.7362741986670898,
+      "grad_norm": 1.4562222001050333,
+      "learning_rate": 4.477677586640854e-05,
+      "loss": 0.5449815988540649,
+      "step": 290,
+      "token_acc": 0.8387928419775553
+    },
+    {
+      "epoch": 0.7388130752142177,
+      "grad_norm": 2.12916421678339,
+      "learning_rate": 4.4700268840168045e-05,
+      "loss": 0.5377695560455322,
+      "step": 291,
+      "token_acc": 0.840630472854641
+    },
+    {
+      "epoch": 0.7413519517613456,
+      "grad_norm": 2.4123707155531,
+      "learning_rate": 4.4623271933713065e-05,
+      "loss": 0.4623599648475647,
+      "step": 292,
+      "token_acc": 0.8558679912443172
+    },
+    {
+      "epoch": 0.7438908283084735,
+      "grad_norm": 2.3146814777436333,
+      "learning_rate": 4.454578706170075e-05,
+      "loss": 0.5539848804473877,
+      "step": 293,
+      "token_acc": 0.8411723411723412
+    },
+    {
+      "epoch": 0.7464297048556015,
+      "grad_norm": 1.9969775879325171,
+      "learning_rate": 4.446781615092235e-05,
+      "loss": 0.48377200961112976,
+      "step": 294,
+      "token_acc": 0.8496886046120182
+    },
+    {
+      "epoch": 0.7489685814027293,
+      "grad_norm": 2.8675177730517296,
+      "learning_rate": 4.4389361140255306e-05,
+      "loss": 0.5668104887008667,
+      "step": 295,
+      "token_acc": 0.8326268464996789
+    },
+    {
+      "epoch": 0.7515074579498572,
+      "grad_norm": 2.7875164416704923,
+      "learning_rate": 4.431042398061499e-05,
+      "loss": 0.4774082601070404,
+      "step": 296,
+      "token_acc": 0.8559006211180125
+    },
+    {
+      "epoch": 0.7540463344969851,
+      "grad_norm": 3.728199688522229,
+      "learning_rate": 4.4231006634906224e-05,
+      "loss": 0.5095118284225464,
+      "step": 297,
+      "token_acc": 0.8488716956802064
+    },
+    {
+      "epoch": 0.756585211044113,
+      "grad_norm": 2.5412776821938974,
+      "learning_rate": 4.415111107797445e-05,
+      "loss": 0.5145753622055054,
+      "step": 298,
+      "token_acc": 0.8456885637923759
+    },
+    {
+      "epoch": 0.7591240875912408,
+      "grad_norm": 3.109090657017562,
+      "learning_rate": 4.407073929655666e-05,
+      "loss": 0.4358251094818115,
+      "step": 299,
+      "token_acc": 0.8618875369944098
+    },
+    {
+      "epoch": 0.7616629641383688,
+      "grad_norm": 1.6297548376808046,
+      "learning_rate": 4.3989893289231954e-05,
+      "loss": 0.4839688539505005,
+      "step": 300,
+      "token_acc": 0.8528946535906359
+    },
+    {
+      "epoch": 0.7642018406854967,
+      "grad_norm": 1.6139429460706416,
+      "learning_rate": 4.3908575066371835e-05,
+      "loss": 0.5315079092979431,
+      "step": 301,
+      "token_acc": 0.8464961067853171
+    },
+    {
+      "epoch": 0.7667407172326246,
+      "grad_norm": 1.958784000024998,
+      "learning_rate": 4.382678665009028e-05,
+      "loss": 0.43339255452156067,
+      "step": 302,
+      "token_acc": 0.8655923195611178
+    },
+    {
+      "epoch": 0.7692795937797524,
+      "grad_norm": 2.9343918370531257,
+      "learning_rate": 4.374453007419336e-05,
+      "loss": 0.4361863434314728,
+      "step": 303,
+      "token_acc": 0.859846630771741
+    },
+    {
+      "epoch": 0.7718184703268803,
+      "grad_norm": 3.237940469376079,
+      "learning_rate": 4.366180738412876e-05,
+      "loss": 0.511203408241272,
+      "step": 304,
+      "token_acc": 0.8461118690313779
+    },
+    {
+      "epoch": 0.7743573468740083,
+      "grad_norm": 2.274147737832287,
+      "learning_rate": 4.357862063693486e-05,
+      "loss": 0.4454347491264343,
+      "step": 305,
+      "token_acc": 0.8601151774103659
+    },
+    {
+      "epoch": 0.7768962234211362,
+      "grad_norm": 4.944015444600801,
+      "learning_rate": 4.34949719011896e-05,
+      "loss": 0.5148607492446899,
+      "step": 306,
+      "token_acc": 0.8461761718183137
+    },
+    {
+      "epoch": 0.779435099968264,
+      "grad_norm": 1.192919570288064,
+      "learning_rate": 4.341086325695905e-05,
+      "loss": 0.4589368402957916,
+      "step": 307,
+      "token_acc": 0.8554825669530066
+    },
+    {
+      "epoch": 0.7819739765153919,
+      "grad_norm": 2.321389755226125,
+      "learning_rate": 4.332629679574566e-05,
+      "loss": 0.4466659128665924,
+      "step": 308,
+      "token_acc": 0.8586858685868587
+    },
+    {
+      "epoch": 0.7845128530625198,
+      "grad_norm": 1.7507551558813503,
+      "learning_rate": 4.324127462043627e-05,
+      "loss": 0.5438239574432373,
+      "step": 309,
+      "token_acc": 0.8434806403064715
+    },
+    {
+      "epoch": 0.7870517296096478,
+      "grad_norm": 1.779981899957501,
+      "learning_rate": 4.3155798845249827e-05,
+      "loss": 0.45011502504348755,
+      "step": 310,
+      "token_acc": 0.8603988603988604
+    },
+    {
+      "epoch": 0.7895906061567757,
+      "grad_norm": 1.1173424722276424,
+      "learning_rate": 4.306987159568479e-05,
+      "loss": 0.474408894777298,
+      "step": 311,
+      "token_acc": 0.8555719678319383
+    },
+    {
+      "epoch": 0.7921294827039035,
+      "grad_norm": 1.6608650795823556,
+      "learning_rate": 4.2983495008466276e-05,
+      "loss": 0.4456779360771179,
+      "step": 312,
+      "token_acc": 0.8610570687418937
+    },
+    {
+      "epoch": 0.7946683592510314,
+      "grad_norm": 1.5083288797918926,
+      "learning_rate": 4.2896671231492966e-05,
+      "loss": 0.4788318872451782,
+      "step": 313,
+      "token_acc": 0.8551842330762639
+    },
+    {
+      "epoch": 0.7972072357981593,
+      "grad_norm": 1.2937780524566131,
+      "learning_rate": 4.2809402423783624e-05,
+      "loss": 0.4659798741340637,
+      "step": 314,
+      "token_acc": 0.8593476531424026
+    },
+    {
+      "epoch": 0.7997461123452873,
+      "grad_norm": 1.0233055916579001,
+      "learning_rate": 4.272169075542348e-05,
+      "loss": 0.5088976621627808,
+      "step": 315,
+      "token_acc": 0.8429627141417534
+    },
+    {
+      "epoch": 0.8022849888924151,
+      "grad_norm": 1.0571728981681126,
+      "learning_rate": 4.263353840751022e-05,
+      "loss": 0.4456738829612732,
+      "step": 316,
+      "token_acc": 0.8666382616105667
+    },
+    {
+      "epoch": 0.804823865439543,
+      "grad_norm": 1.8198326747222258,
+      "learning_rate": 4.254494757209979e-05,
+      "loss": 0.4678633213043213,
+      "step": 317,
+      "token_acc": 0.8546817674270218
+    },
+    {
+      "epoch": 0.8073627419866709,
+      "grad_norm": 1.5377357010824237,
+      "learning_rate": 4.245592045215182e-05,
+      "loss": 0.5010336637496948,
+      "step": 318,
+      "token_acc": 0.8465278833004103
+    },
+    {
+      "epoch": 0.8099016185337988,
+      "grad_norm": 1.4181630525718223,
+      "learning_rate": 4.2366459261474933e-05,
+      "loss": 0.5233482718467712,
+      "step": 319,
+      "token_acc": 0.843609022556391
+    },
+    {
+      "epoch": 0.8124404950809266,
+      "grad_norm": 2.7968517592507247,
+      "learning_rate": 4.227656622467162e-05,
+      "loss": 0.45297539234161377,
+      "step": 320,
+      "token_acc": 0.8587121852290012
+    },
+    {
+      "epoch": 0.8149793716280546,
+      "grad_norm": 3.9632559263367324,
+      "learning_rate": 4.2186243577082954e-05,
+      "loss": 0.49113717675209045,
+      "step": 321,
+      "token_acc": 0.8499593165174939
+    },
+    {
+      "epoch": 0.8175182481751825,
+      "grad_norm": 1.5212678599870548,
+      "learning_rate": 4.2095493564733005e-05,
+      "loss": 0.4760684370994568,
+      "step": 322,
+      "token_acc": 0.8581730769230769
+    },
+    {
+      "epoch": 0.8200571247223104,
+      "grad_norm": 1.1979894810329197,
+      "learning_rate": 4.2004318444272985e-05,
+      "loss": 0.5287943482398987,
+      "step": 323,
+      "token_acc": 0.8400316247199895
+    },
+    {
+      "epoch": 0.8225960012694383,
+      "grad_norm": 13.231283362213604,
+      "learning_rate": 4.191272048292513e-05,
+      "loss": 0.45438772439956665,
+      "step": 324,
+      "token_acc": 0.8593374939990398
+    },
+    {
+      "epoch": 0.8251348778165661,
+      "grad_norm": 1.9566355667367066,
+      "learning_rate": 4.1820701958426325e-05,
+      "loss": 0.4688713550567627,
+      "step": 325,
+      "token_acc": 0.8610249593856151
+    },
+    {
+      "epoch": 0.8276737543636941,
+      "grad_norm": 4.323839382951773,
+      "learning_rate": 4.172826515897146e-05,
+      "loss": 0.5428496599197388,
+      "step": 326,
+      "token_acc": 0.841347459929367
+    },
+    {
+      "epoch": 0.830212630910822,
+      "grad_norm": 2.713639671675627,
+      "learning_rate": 4.163541238315653e-05,
+      "loss": 0.5651642680168152,
+      "step": 327,
+      "token_acc": 0.8335643015521065
+    },
+    {
+      "epoch": 0.8327515074579499,
+      "grad_norm": 2.2005640892547205,
+      "learning_rate": 4.154214593992149e-05,
+      "loss": 0.42816388607025146,
+      "step": 328,
+      "token_acc": 0.8655292606861127
+    },
+    {
+      "epoch": 0.8352903840050777,
+      "grad_norm": 1.7693852546194808,
+      "learning_rate": 4.144846814849282e-05,
+      "loss": 0.5406675934791565,
+      "step": 329,
+      "token_acc": 0.8363126015659624
+    },
+    {
+      "epoch": 0.8378292605522056,
+      "grad_norm": 1.2824936447977062,
+      "learning_rate": 4.1354381338325864e-05,
+      "loss": 0.47528088092803955,
+      "step": 330,
+      "token_acc": 0.8529364953047907
+    },
+    {
+      "epoch": 0.8403681370993336,
+      "grad_norm": 2.1290755841224964,
+      "learning_rate": 4.1259887849046906e-05,
+      "loss": 0.47919371724128723,
+      "step": 331,
+      "token_acc": 0.8517776664997496
+    },
+    {
+      "epoch": 0.8429070136464615,
+      "grad_norm": 1.7923254144391947,
+      "learning_rate": 4.116499003039499e-05,
+      "loss": 0.4583389163017273,
+      "step": 332,
+      "token_acc": 0.8578696865255211
+    },
+    {
+      "epoch": 0.8454458901935893,
+      "grad_norm": 3.831696945491014,
+      "learning_rate": 4.1069690242163484e-05,
+      "loss": 0.4751017093658447,
+      "step": 333,
+      "token_acc": 0.8565161762017912
+    },
+    {
+      "epoch": 0.8479847667407172,
+      "grad_norm": 1.3534101044854359,
+      "learning_rate": 4.09739908541414e-05,
+      "loss": 0.5170926451683044,
+      "step": 334,
+      "token_acc": 0.844575246579701
+    },
+    {
+      "epoch": 0.8505236432878451,
+      "grad_norm": 1.7263471686071763,
+      "learning_rate": 4.087789424605447e-05,
+      "loss": 0.560020923614502,
+      "step": 335,
+      "token_acc": 0.8369974874371859
+    },
+    {
+      "epoch": 0.853062519834973,
+      "grad_norm": 1.976895260311541,
+      "learning_rate": 4.078140280750597e-05,
+      "loss": 0.48105573654174805,
+      "step": 336,
+      "token_acc": 0.8486519607843137
+    },
+    {
+      "epoch": 0.8556013963821009,
+      "grad_norm": 1.3618234100955324,
+      "learning_rate": 4.0684518937917315e-05,
+      "loss": 0.535722553730011,
+      "step": 337,
+      "token_acc": 0.8411811652035116
+    },
+    {
+      "epoch": 0.8581402729292288,
+      "grad_norm": 1.7410934714261248,
+      "learning_rate": 4.058724504646834e-05,
+      "loss": 0.5101606845855713,
+      "step": 338,
+      "token_acc": 0.8413575374901342
+    },
+    {
+      "epoch": 0.8606791494763567,
+      "grad_norm": 1.6462446903462482,
+      "learning_rate": 4.048958355203746e-05,
+      "loss": 0.4928050637245178,
+      "step": 339,
+      "token_acc": 0.8460967646185812
+    },
+    {
+      "epoch": 0.8632180260234846,
+      "grad_norm": 1.228936011236623,
+      "learning_rate": 4.039153688314145e-05,
+      "loss": 0.5174915194511414,
+      "step": 340,
+      "token_acc": 0.8437601824698598
+    },
+    {
+      "epoch": 0.8657569025706126,
+      "grad_norm": 1.4612143586664255,
+      "learning_rate": 4.029310747787516e-05,
+      "loss": 0.41705322265625,
+      "step": 341,
+      "token_acc": 0.8653099269846202
+    },
+    {
+      "epoch": 0.8682957791177404,
+      "grad_norm": 1.797059242292307,
+      "learning_rate": 4.0194297783850755e-05,
+      "loss": 0.5154389142990112,
+      "step": 342,
+      "token_acc": 0.8439297831538575
+    },
+    {
+      "epoch": 0.8708346556648683,
+      "grad_norm": 2.068548395822149,
+      "learning_rate": 4.009511025813694e-05,
+      "loss": 0.45814093947410583,
+      "step": 343,
+      "token_acc": 0.8591615638247763
+    },
+    {
+      "epoch": 0.8733735322119962,
+      "grad_norm": 2.106347854495477,
+      "learning_rate": 3.9995547367197845e-05,
+      "loss": 0.5106515884399414,
+      "step": 344,
+      "token_acc": 0.8444444444444444
+    },
+    {
+      "epoch": 0.8759124087591241,
+      "grad_norm": 2.277599588151966,
+      "learning_rate": 3.9895611586831685e-05,
+      "loss": 0.5230048894882202,
+      "step": 345,
+      "token_acc": 0.8413416536661467
+    },
+    {
+      "epoch": 0.8784512853062519,
+      "grad_norm": 2.12251688559056,
+      "learning_rate": 3.9795305402109195e-05,
+      "loss": 0.4795979857444763,
+      "step": 346,
+      "token_acc": 0.8566383257030739
+    },
+    {
+      "epoch": 0.8809901618533799,
+      "grad_norm": 6.771491520442473,
+      "learning_rate": 3.969463130731183e-05,
+      "loss": 0.41639000177383423,
+      "step": 347,
+      "token_acc": 0.869641912153819
+    },
+    {
+      "epoch": 0.8835290384005078,
+      "grad_norm": 8.929919537256396,
+      "learning_rate": 3.959359180586975e-05,
+      "loss": 0.4425828158855438,
+      "step": 348,
+      "token_acc": 0.8597887647328946
+    },
+    {
+      "epoch": 0.8860679149476357,
+      "grad_norm": 7.530785543727866,
+      "learning_rate": 3.9492189410299566e-05,
+      "loss": 0.44385579228401184,
+      "step": 349,
+      "token_acc": 0.865767878077374
+    },
+    {
+      "epoch": 0.8886067914947635,
+      "grad_norm": 2.6343520915908716,
+      "learning_rate": 3.939042664214184e-05,
+      "loss": 0.48342394828796387,
+      "step": 350,
+      "token_acc": 0.8567626018965231
+    },
+    {
+      "epoch": 0.8911456680418914,
+      "grad_norm": 1.5801412806386852,
+      "learning_rate": 3.928830603189844e-05,
+      "loss": 0.4776611030101776,
+      "step": 351,
+      "token_acc": 0.8521291530182499
+    },
+    {
+      "epoch": 0.8936845445890194,
+      "grad_norm": 3.7389218036878478,
+      "learning_rate": 3.918583011896955e-05,
+      "loss": 0.45217645168304443,
+      "step": 352,
+      "token_acc": 0.8585561910650796
+    },
+    {
+      "epoch": 0.8962234211361473,
+      "grad_norm": 1.6890575760456612,
+      "learning_rate": 3.908300145159055e-05,
+      "loss": 0.5242031216621399,
+      "step": 353,
+      "token_acc": 0.8453546152687849
+    },
+    {
+      "epoch": 0.8987622976832752,
+      "grad_norm": 1.992965354554827,
+      "learning_rate": 3.897982258676867e-05,
+      "loss": 0.4830443859100342,
+      "step": 354,
+      "token_acc": 0.8554014932195643
+    },
+    {
+      "epoch": 0.901301174230403,
+      "grad_norm": 1.2964405728485144,
+      "learning_rate": 3.887629609021938e-05,
+      "loss": 0.4663028419017792,
+      "step": 355,
+      "token_acc": 0.8561201572150478
+    },
+    {
+      "epoch": 0.9038400507775309,
+      "grad_norm": 8.39081588467176,
+      "learning_rate": 3.8772424536302564e-05,
+      "loss": 0.5014960169792175,
+      "step": 356,
+      "token_acc": 0.8504262120404902
+    },
+    {
+      "epoch": 0.9063789273246589,
+      "grad_norm": 1.8861771407475878,
+      "learning_rate": 3.866821050795859e-05,
+      "loss": 0.5827922821044922,
+      "step": 357,
+      "token_acc": 0.8322280231409623
+    },
+    {
+      "epoch": 0.9089178038717868,
+      "grad_norm": 1.631982885362827,
+      "learning_rate": 3.856365659664399e-05,
+      "loss": 0.5254430770874023,
+      "step": 358,
+      "token_acc": 0.8455894342083808
+    },
+    {
+      "epoch": 0.9114566804189146,
+      "grad_norm": 1.6184938593440648,
+      "learning_rate": 3.845876540226706e-05,
+      "loss": 0.5034159421920776,
+      "step": 359,
+      "token_acc": 0.8477827221114009
+    },
+    {
+      "epoch": 0.9139955569660425,
+      "grad_norm": 2.616866018868145,
+      "learning_rate": 3.835353953312322e-05,
+      "loss": 0.4409644603729248,
+      "step": 360,
+      "token_acc": 0.8624558168126633
+    },
+    {
+      "epoch": 0.9165344335131704,
+      "grad_norm": 1.1082161386999507,
+      "learning_rate": 3.824798160583012e-05,
+      "loss": 0.4826850891113281,
+      "step": 361,
+      "token_acc": 0.8542769573697409
+    },
+    {
+      "epoch": 0.9190733100602984,
+      "grad_norm": 1.4706520431007584,
+      "learning_rate": 3.814209424526262e-05,
+      "loss": 0.5543685555458069,
+      "step": 362,
+      "token_acc": 0.8411547002220577
+    },
+    {
+      "epoch": 0.9216121866074262,
+      "grad_norm": 1.4937567660832907,
+      "learning_rate": 3.803588008448745e-05,
+      "loss": 0.46426546573638916,
+      "step": 363,
+      "token_acc": 0.8584509156409487
+    },
+    {
+      "epoch": 0.9241510631545541,
+      "grad_norm": 1.164319366810129,
+      "learning_rate": 3.7929341764697816e-05,
+      "loss": 0.43454286456108093,
+      "step": 364,
+      "token_acc": 0.869382677655108
+    },
+    {
+      "epoch": 0.926689939701682,
+      "grad_norm": 1.4647461875367664,
+      "learning_rate": 3.782248193514766e-05,
+      "loss": 0.40108054876327515,
+      "step": 365,
+      "token_acc": 0.8742168674698795
+    },
+    {
+      "epoch": 0.9292288162488099,
+      "grad_norm": 1.6339280778635523,
+      "learning_rate": 3.771530325308579e-05,
+      "loss": 0.48394569754600525,
+      "step": 366,
+      "token_acc": 0.8536367122587243
+    },
+    {
+      "epoch": 0.9317676927959379,
+      "grad_norm": 1.4990029947108798,
+      "learning_rate": 3.7607808383689856e-05,
+      "loss": 0.5164212584495544,
+      "step": 367,
+      "token_acc": 0.8487874465049928
+    },
+    {
+      "epoch": 0.9343065693430657,
+      "grad_norm": 1.246505387725808,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 0.5112602710723877,
+      "step": 368,
+      "token_acc": 0.8492902446390819
+    },
+    {
+      "epoch": 0.9368454458901936,
+      "grad_norm": 3.107413430389552,
+      "learning_rate": 3.739188078285244e-05,
+      "loss": 0.4383936822414398,
+      "step": 369,
+      "token_acc": 0.8606896551724138
+    },
+    {
+      "epoch": 0.9393843224373215,
+      "grad_norm": 37.635995940969075,
+      "learning_rate": 3.7283453420812786e-05,
+      "loss": 0.46092715859413147,
+      "step": 370,
+      "token_acc": 0.8597212294496068
+    },
+    {
+      "epoch": 0.9419231989844494,
+      "grad_norm": 3.577937521692329,
+      "learning_rate": 3.717472061010918e-05,
+      "loss": 0.5413942337036133,
+      "step": 371,
+      "token_acc": 0.8350426115845732
+    },
+    {
+      "epoch": 0.9444620755315772,
+      "grad_norm": 5.47409206936157,
+      "learning_rate": 3.706568505456527e-05,
+      "loss": 0.48648974299430847,
+      "step": 372,
+      "token_acc": 0.8585890540968049
+    },
+    {
+      "epoch": 0.9470009520787052,
+      "grad_norm": 1.4741286710796073,
+      "learning_rate": 3.695634946553296e-05,
+      "loss": 0.47160500288009644,
+      "step": 373,
+      "token_acc": 0.8502487562189055
+    },
+    {
+      "epoch": 0.9495398286258331,
+      "grad_norm": 2.090944147666011,
+      "learning_rate": 3.6846716561824965e-05,
+      "loss": 0.5264440774917603,
+      "step": 374,
+      "token_acc": 0.84245960502693
+    },
+    {
+      "epoch": 0.952078705172961,
+      "grad_norm": 5.396217963829348,
+      "learning_rate": 3.673678906964727e-05,
+      "loss": 0.46441876888275146,
+      "step": 375,
+      "token_acc": 0.8582541054451167
+    },
+    {
+      "epoch": 0.9546175817200888,
+      "grad_norm": 2.5487410815491325,
+      "learning_rate": 3.662656972253127e-05,
+      "loss": 0.4945961534976959,
+      "step": 376,
+      "token_acc": 0.849025974025974
+    },
+    {
+      "epoch": 0.9571564582672167,
+      "grad_norm": 1.4487108456202973,
+      "learning_rate": 3.651606126126581e-05,
+      "loss": 0.44490116834640503,
+      "step": 377,
+      "token_acc": 0.8634252090430474
+    },
+    {
+      "epoch": 0.9596953348143447,
+      "grad_norm": 2.4501839034153567,
+      "learning_rate": 3.6405266433829075e-05,
+      "loss": 0.45414626598358154,
+      "step": 378,
+      "token_acc": 0.8611764705882353
+    },
+    {
+      "epoch": 0.9622342113614726,
+      "grad_norm": 2.498672242292354,
+      "learning_rate": 3.6294187995320214e-05,
+      "loss": 0.4957122206687927,
+      "step": 379,
+      "token_acc": 0.8477234401349073
+    },
+    {
+      "epoch": 0.9647730879086004,
+      "grad_norm": 1.5923271833585566,
+      "learning_rate": 3.6182828707890816e-05,
+      "loss": 0.5208961963653564,
+      "step": 380,
+      "token_acc": 0.8432196541134637
+    },
+    {
+      "epoch": 0.9673119644557283,
+      "grad_norm": 4.277533304496898,
+      "learning_rate": 3.607119134067629e-05,
+      "loss": 0.5402977466583252,
+      "step": 381,
+      "token_acc": 0.8438965563446336
+    },
+    {
+      "epoch": 0.9698508410028562,
+      "grad_norm": 1.391785836034451,
+      "learning_rate": 3.5959278669726935e-05,
+      "loss": 0.5133845210075378,
+      "step": 382,
+      "token_acc": 0.8467186095295175
+    },
+    {
+      "epoch": 0.9723897175499842,
+      "grad_norm": 1.6652300791673929,
+      "learning_rate": 3.5847093477938956e-05,
+      "loss": 0.4607965350151062,
+      "step": 383,
+      "token_acc": 0.8581548321808062
+    },
+    {
+      "epoch": 0.9749285940971121,
+      "grad_norm": 3.047516730060917,
+      "learning_rate": 3.5734638554985236e-05,
+      "loss": 0.558885931968689,
+      "step": 384,
+      "token_acc": 0.8354353441726181
+    },
+    {
+      "epoch": 0.9774674706442399,
+      "grad_norm": 1.7394548434827772,
+      "learning_rate": 3.562191669724597e-05,
+      "loss": 0.45114433765411377,
+      "step": 385,
+      "token_acc": 0.8557377049180328
+    },
+    {
+      "epoch": 0.9800063471913678,
+      "grad_norm": 2.3715567586043527,
+      "learning_rate": 3.550893070773914e-05,
+      "loss": 0.4739205837249756,
+      "step": 386,
+      "token_acc": 0.8504549457000293
+    },
+    {
+      "epoch": 0.9825452237384957,
+      "grad_norm": 2.7051039254253944,
+      "learning_rate": 3.5395683396050825e-05,
+      "loss": 0.47992223501205444,
+      "step": 387,
+      "token_acc": 0.8514475557664927
+    },
+    {
+      "epoch": 0.9850841002856237,
+      "grad_norm": 2.9508108026603526,
+      "learning_rate": 3.5282177578265296e-05,
+      "loss": 0.41233572363853455,
+      "step": 388,
+      "token_acc": 0.8679525222551929
+    },
+    {
+      "epoch": 0.9876229768327515,
+      "grad_norm": 1.4813660303012237,
+      "learning_rate": 3.516841607689501e-05,
+      "loss": 0.5352930426597595,
+      "step": 389,
+      "token_acc": 0.8431159981415518
+    },
+    {
+      "epoch": 0.9901618533798794,
+      "grad_norm": 3.1303557072936457,
+      "learning_rate": 3.505440172081044e-05,
+      "loss": 0.4628872275352478,
+      "step": 390,
+      "token_acc": 0.86190833959429
+    },
+    {
+      "epoch": 0.9927007299270073,
+      "grad_norm": 1.4674915872936216,
+      "learning_rate": 3.494013734516971e-05,
+      "loss": 0.4559894800186157,
+      "step": 391,
+      "token_acc": 0.8610082138830729
+    },
+    {
+      "epoch": 0.9952396064741352,
+      "grad_norm": 1.953348901842891,
+      "learning_rate": 3.4825625791348096e-05,
+      "loss": 0.4409753978252411,
+      "step": 392,
+      "token_acc": 0.862080370431619
+    },
+    {
+      "epoch": 0.997778483021263,
+      "grad_norm": 4.310074710886207,
+      "learning_rate": 3.471086990686737e-05,
+      "loss": 0.4559399485588074,
+      "step": 393,
+      "token_acc": 0.8591592988987126
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.7219046332106547,
+      "learning_rate": 3.459587254532502e-05,
+      "loss": 0.48068922758102417,
+      "step": 394,
+      "token_acc": 0.8567295770767324
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.38288021087646484,
+      "eval_runtime": 8.607,
+      "eval_samples_per_second": 1.394,
+      "eval_steps_per_second": 0.349,
+      "eval_token_acc": 0.8742701440249124,
+      "step": 394
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 788,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2082783438045184.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:127702dae975ef36e1701c2735de30b8da42177c8aab9767eeec06a55c650d06
+size 9873

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-394/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/README.md

@@ -0,0 +1,207 @@
+---
+base_model: /data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.19.1

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/adapter_config.json

@@ -0,0 +1,40 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [],
+  "peft_type": "LORA",
+  "peft_version": "0.19.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "^(model\\.language_model(?=\\.).*\\.(up_proj|o_proj|down_proj|q_proj|v_proj|gate_proj|k_proj)|(?!(model.visual.merger))model\\.visual(?=\\.).*\\.(qkv|up_proj|mlp.2|down_proj|mlp.0|attn.proj|gate_proj))$",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfcd28130097822de7163f52b0875e80ba705fe726b822e4ff77d63c502826cd
+size 103145560

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/additional_config.json

@@ -0,0 +1 @@
+{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/args.json

@@ -0,0 +1,403 @@
+{
+  "output_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "epoch",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 5e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 2.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.2,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 1,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "epoch",
+  "save_steps": 500,
+  "save_total_limit": null,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "seed": 42,
+  "data_seed": 42,
+  "jit_mode_eval": false,
+  "bf16": true,
+  "fp16": false,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "ddp_backend": null,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 2000.0,
+  "dataloader_num_workers": 48,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057",
+  "disable_tqdm": null,
+  "remove_unused_columns": true,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": [],
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "parallelism_config": null,
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch_fused",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "project": "huggingface",
+  "trackio_space_id": "trackio",
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_token": null,
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "hub_revision": null,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "ddp_timeout": 18000000,
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": false,
+  "liger_kernel_config": null,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": true,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "tuner_backend": "peft",
+  "vit_gradient_checkpointing": true,
+  "router_aux_loss_coef": 0.0,
+  "enable_dft_loss": false,
+  "enable_channel_loss": false,
+  "safe_serialization": true,
+  "max_shard_size": "5GB",
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "use_logits_to_keep": null,
+  "ds3_gather_for_generation": true,
+  "resume_only_model": false,
+  "optimizer": null,
+  "loss_type": null,
+  "eval_metric": null,
+  "callbacks": [],
+  "early_stop_interval": null,
+  "eval_use_evalscope": false,
+  "eval_dataset": [],
+  "eval_dataset_args": null,
+  "eval_limit": null,
+  "eval_generation_config": null,
+  "extra_eval_args": null,
+  "tuner_type": "lora",
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "use_flash_ckpt": false,
+  "use_ray": false,
+  "ray_exp_name": null,
+  "device_groups": null,
+  "model": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "model_type": "qwen2_5_vl",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "bfloat16",
+  "attn_impl": null,
+  "experts_impl": null,
+  "new_special_tokens": [],
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "max_model_len": null,
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen2_5_vl",
+  "system": null,
+  "max_length": 16240,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_side": "right",
+  "padding_free": false,
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "template_backend": "swift",
+  "response_prefix": null,
+  "enable_thinking": null,
+  "add_non_thinking_prefix": true,
+  "dataset": [
+    "data.jsonl"
+  ],
+  "val_dataset": [],
+  "cached_dataset": [],
+  "cached_val_dataset": [],
+  "split_dataset_ratio": 0.001,
+  "dataset_num_proc": 100,
+  "load_from_cache_file": false,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "model_name": null,
+  "model_author": null,
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "bfloat16",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "structured_outputs_regex": null,
+  "adapters": [],
+  "external_plugins": [],
+  "custom_register_path": [],
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "packing": false,
+  "packing_length": null,
+  "packing_num_proc": 1,
+  "lazy_tokenize": true,
+  "use_hf": false,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": false,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "target_parameters": null,
+  "modules_to_save": [],
+  "lora_rank": 16,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": "ms-swift",
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_notification_method": null,
+  "swanlab_webhook_url": null,
+  "swanlab_secret": null,
+  "swanlab_sender_email": null,
+  "swanlab_receiver_email": null,
+  "swanlab_smtp_server": null,
+  "swanlab_smtp_port": null,
+  "swanlab_email_language": "zh",
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "create_checkpoint_symlink": false,
+  "zero_hpz_partition_size": null,
+  "deepspeed_autotp_size": null,
+  "swift_version": "4.1.3",
+  "ckpt_dir": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "Qwen2.5-VL-7B-Instruct",
+  "model_info": "ModelInfo(model_type='qwen2_5_vl', model_dir='/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct', torch_dtype=torch.bfloat16, max_model_len=128000, quant_method=None, quant_bits=None, rope_scaling={'type': 'default', 'mrope_section': [16, 24, 24], 'rope_type': 'default'}, is_moe_model=False, is_multimodal=True, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen2_5_vl', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-3B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-7B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-32B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', hf_model_id='Qwen/Qwen2.5-VL-72B-Instruct-AWQ', model_path=None, ms_revision=None, hf_revision=None)], template='qwen2_5_vl', ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', hf_model_id='XiaomiMiMo/MiMo-VL-7B-SFT', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='XiaomiMiMo/MiMo-VL-7B-RL', hf_model_id='XiaomiMiMo/MiMo-VL-7B-RL', model_path=None, ms_revision=None, hf_revision=None)], template='mimo_vl', ignore_patterns=None, requires=None, tags=[])], loader=<class 'swift.model.models.qwen.Qwen2_5VLLoader'>, template='qwen2_5_vl', model_arch=MultiModelKeys(arch_name='qwen2_vl', embedding=None, module_list=None, lm_head=None, q_proj=None, k_proj=None, v_proj=None, o_proj=None, attention=None, mlp=None, down_proj=None, qkv_proj=None, qk_proj=None, qa_proj=None, qb_proj=None, kv_proj=None, kva_proj=None, kvb_proj=None, language_model=['model.language_model', 'lm_head'], aligner=['model.visual.merger'], vision_tower=['model.visual'], generator=[]), mcore_model_type=None, architectures=['Qwen2_5_VLForConditionalGeneration'], additional_saved_files=[], torch_dtype=None, is_multimodal=True, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'], tags=[])",
+  "model_dir": "/data/yutao/lzt/BrowserAgent_v2/models/Qwen2.5-VL-7B-Instruct",
+  "template_meta": "QwenTemplateMeta(template_type='qwen2_5_vl', prefix=[], prompt=['<|im_start|>user\\n{{QUERY}}<|im_end|>\\n<|im_start|>assistant\\n'], chat_sep=['<|im_end|>\\n'], suffix=['<|im_end|>\\n'], template_cls=<class 'swift.template.templates.qwen.Qwen2_5VLTemplate'>, system_prefix=['<|im_start|>system\\n{{SYSTEM}}<|im_end|>\\n'], default_system='You are a helpful assistant.', auto_add_bos=False, stop_words=['<|endoftext|>'], agent_template='hermes', is_thinking=False, thinking_prefix='', non_thinking_prefix='', history_thinking_prefix='')",
+  "_val_dataset_exists": true,
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "epoch",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.EPOCH: 'epoch'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=5e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=2.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.2, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.EPOCH: 'epoch'>, save_steps=500, save_total_limit=None, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=2000.0, dataloader_num_workers=48, dataloader_prefetch_factor=2, past_index=-1, run_name='/data/yutao/lzt/BrowserAgent_v2/sft/output/Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], project='huggingface', trackio_space_id='trackio', ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=False, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, safe_serialization=True, max_shard_size='5GB', check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, eval_metric=None, callbacks=[], early_stop_interval=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, tuner_type='lora', use_galore=False, galore_target_modules=None, galore_rank=128, galore_update_proj_gap=50, galore_scale=1.0, galore_proj_type='std', galore_optim_per_parameter=False, galore_with_embedding=False, galore_quantization=False, galore_proj_quant=False, galore_proj_bits=4, galore_proj_group_size=256, galore_cos_threshold=0.4, galore_gamma_proj=2, galore_queue_size=5, lisa_activated_layers=0, lisa_step_interval=20, use_flash_ckpt=False)"
+}

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8acec1076365a7b99766f89c3a33df0119f303fcce39c2376ae687248780991
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ec9e7d570948c6b5a1bd2d13f88b5d9e3786bead128ee45ae5ec7183255edd1
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b8d8d40a736e470d23022261733a5aa65ed256d7f19d67f4c6896895d7104ba
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31d79581ff29eec6a19fb84706938056987171ebcd37f626a2a2c757735fe5aa
+size 154569157

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b781e914afe6c9319e94bff5acb2aebe2cfd0e6c6800f0c848594737f10bf11
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0946b7cb82f6053e42e0a427494eb666966bd43eab9d245f11dd90b732ba7acb
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b826c199505a59dddcc29e910e7d9b0c8cd86223c3bee451f5d4e90264e98a
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/global_step788/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce2e90621ddf30a1cb61bc8edeea4c823e74076e691f1f2dd0f4d995ad8c6fca
+size 766241

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/latest

@@ -0,0 +1 @@
+global_step788

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22d6790b00b5718921d1c1f3fd3cf361895b4d3db078805589fbef73a26d446a
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98b9ab57254e7912ec73442baaca5373adde75dda800dd0388800a5ce6645908
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05777f4a94779fcb4f7d833dd95f50147032e0dd69503ada716e8999b69e3e21
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d049d405eb58f9feec0c323caaf3809ab89c1cc9d844b91a5839c835a2af754d
+size 15429

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acc9808b873dd2b1009a6d63db076a42434e830983edcd704445fb5b060787f4
+size 1465

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:127702dae975ef36e1701c2735de30b8da42177c8aab9767eeec06a55c650d06
+size 9873

Qwen2.5-VL-7B-Instruct-task-opsrc_12619-sft-5e-5lr-freeze_false/v0-20260503-074057/checkpoint-788/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info("Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info("Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)