Upload folder using huggingface_hub

v0-20250526-142723/args.json

@@ -0,0 +1,373 @@
+{
+  "model": "/kaggle/input/qwen-3/transformers/32b-awq/1",
+  "model_type": "qwen3",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "float16",
+  "attn_impl": "flash_attn",
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen3",
+  "system": null,
+  "max_length": 512,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "combined_messages.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "data_seed": 42,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": true,
+  "model_name": [
+    null,
+    null
+  ],
+  "model_author": [
+    null,
+    null
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "float32",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "lora",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": false,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ddp_timeout": 1800,
+  "ddp_backend": null,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/kaggle/tmp/save_checkpoint/v0-20250526-142723",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "steps",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 3e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 3.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/kaggle/tmp/save_checkpoint/v0-20250526-142723/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "steps",
+  "save_steps": 640.0,
+  "save_total_limit": 4,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": false,
+  "fp16": true,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 640.0,
+  "dataloader_num_workers": 8,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/kaggle/tmp/save_checkpoint/v0-20250526-142723",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": true,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "vit_gradient_checkpointing": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "optimizer": null,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 32,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": true,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "1",
+  "model_info": "ModelInfo(model_type='qwen3', model_dir='/kaggle/input/qwen-3/transformers/32b-awq/1', torch_dtype=torch.float16, max_model_len=40960, quant_method='awq', quant_bits=4, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen3', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-0.6B-Base', hf_model_id='Qwen/Qwen3-0.6B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-Base', hf_model_id='Qwen/Qwen3-1.7B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-Base', hf_model_id='Qwen/Qwen3-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-Base', hf_model_id='Qwen/Qwen3-8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-Base', hf_model_id='Qwen/Qwen3-14B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-Base', hf_model_id='Qwen/Qwen3-32B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B', hf_model_id='Qwen/Qwen3-0.6B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B', hf_model_id='Qwen/Qwen3-1.7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B', hf_model_id='Qwen/Qwen3-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B', hf_model_id='Qwen/Qwen3-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B', hf_model_id='Qwen/Qwen3-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B', hf_model_id='Qwen/Qwen3-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B-FP8', hf_model_id='Qwen/Qwen3-0.6B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-FP8', hf_model_id='Qwen/Qwen3-1.7B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-FP8', hf_model_id='Qwen/Qwen3-4B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-FP8', hf_model_id='Qwen/Qwen3-8B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-FP8', hf_model_id='Qwen/Qwen3-14B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-FP8', hf_model_id='Qwen/Qwen3-32B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-AWQ', hf_model_id='Qwen/Qwen3-4B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-AWQ', hf_model_id='Qwen/Qwen3-8B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-AWQ', hf_model_id='Qwen/Qwen3-14B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-AWQ', hf_model_id='Qwen/Qwen3-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='swift/Qwen3-32B-AWQ', hf_model_id=None, model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen3', get_function=<function get_model_tokenizer_with_flash_attn at 0x7e1c95bd7880>, model_arch='llama', architectures=['Qwen3ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.51'], tags=[])",
+  "model_dir": "/kaggle/input/qwen-3/transformers/32b-awq/1",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/kaggle/tmp/save_checkpoint/v0-20250526-142723', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=3e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/kaggle/tmp/save_checkpoint/v0-20250526-142723/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=640, save_total_limit=4, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=False, fp16=True, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=640, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/kaggle/tmp/save_checkpoint/v0-20250526-142723', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=True, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, vit_gradient_checkpointing=True, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='lora', local_repo_path=None, galore_config=None)"
+}

v0-20250526-142723/checkpoint-640/README.md

@@ -0,0 +1,202 @@
+---
+base_model: /kaggle/input/qwen-3/transformers/32b-awq/1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

v0-20250526-142723/checkpoint-640/adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "/kaggle/input/qwen-3/transformers/32b-awq/1",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [],
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "down_proj",
+    "up_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

v0-20250526-142723/checkpoint-640/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42d591d4dedc9f27ecafa9089762ccfec480789034e044a5c390f866c8c1f994
+size 536991984

v0-20250526-142723/checkpoint-640/additional_config.json

@@ -0,0 +1 @@
+{"lora_dtype": null, "lorap_lr_ratio": null, "lorap_emb_lr": 1e-06}

v0-20250526-142723/checkpoint-640/args.json

@@ -0,0 +1,373 @@
+{
+  "model": "/kaggle/input/qwen-3/transformers/32b-awq/1",
+  "model_type": "qwen3",
+  "model_revision": null,
+  "task_type": "causal_lm",
+  "torch_dtype": "float16",
+  "attn_impl": "flash_attn",
+  "num_labels": null,
+  "problem_type": null,
+  "rope_scaling": null,
+  "device_map": null,
+  "max_memory": {},
+  "local_repo_path": null,
+  "init_strategy": null,
+  "template": "qwen3",
+  "system": null,
+  "max_length": 512,
+  "truncation_strategy": "delete",
+  "max_pixels": null,
+  "agent_template": null,
+  "norm_bbox": null,
+  "use_chat_template": true,
+  "padding_free": false,
+  "padding_side": "right",
+  "loss_scale": "default",
+  "sequence_parallel_size": 1,
+  "response_prefix": null,
+  "template_backend": "swift",
+  "dataset": [
+    "combined_messages.jsonl"
+  ],
+  "val_dataset": [],
+  "split_dataset_ratio": 0.0,
+  "data_seed": 42,
+  "dataset_num_proc": 8,
+  "load_from_cache_file": true,
+  "dataset_shuffle": true,
+  "val_dataset_shuffle": false,
+  "streaming": false,
+  "interleave_prob": null,
+  "stopping_strategy": "first_exhausted",
+  "shuffle_buffer_size": 1000,
+  "download_mode": "reuse_dataset_if_exists",
+  "columns": {},
+  "strict": false,
+  "remove_unused_columns": true,
+  "model_name": [
+    null,
+    null
+  ],
+  "model_author": [
+    null,
+    null
+  ],
+  "custom_dataset_info": [],
+  "quant_method": null,
+  "quant_bits": null,
+  "hqq_axis": null,
+  "bnb_4bit_compute_dtype": "float32",
+  "bnb_4bit_quant_type": "nf4",
+  "bnb_4bit_use_double_quant": true,
+  "bnb_4bit_quant_storage": null,
+  "max_new_tokens": 64,
+  "temperature": 0.0,
+  "top_k": null,
+  "top_p": null,
+  "repetition_penalty": null,
+  "num_beams": 1,
+  "stream": false,
+  "stop_words": [],
+  "logprobs": false,
+  "top_logprobs": null,
+  "ckpt_dir": null,
+  "lora_modules": [],
+  "tuner_backend": "peft",
+  "train_type": "lora",
+  "adapters": [],
+  "external_plugins": [],
+  "seed": 42,
+  "model_kwargs": {},
+  "load_args": false,
+  "load_data_args": false,
+  "use_hf": false,
+  "hub_token": null,
+  "custom_register_path": [],
+  "ddp_timeout": 1800,
+  "ddp_backend": null,
+  "ignore_args_error": false,
+  "use_swift_lora": false,
+  "output_dir": "/kaggle/tmp/save_checkpoint/v0-20250526-142723",
+  "overwrite_output_dir": false,
+  "do_train": false,
+  "do_eval": false,
+  "do_predict": false,
+  "eval_strategy": "steps",
+  "prediction_loss_only": false,
+  "per_device_train_batch_size": 1,
+  "per_device_eval_batch_size": 1,
+  "per_gpu_train_batch_size": null,
+  "per_gpu_eval_batch_size": null,
+  "gradient_accumulation_steps": 8,
+  "eval_accumulation_steps": null,
+  "eval_delay": 0,
+  "torch_empty_cache_steps": null,
+  "learning_rate": 3e-05,
+  "weight_decay": 0.1,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 3.0,
+  "max_steps": -1,
+  "lr_scheduler_type": "cosine",
+  "lr_scheduler_kwargs": null,
+  "warmup_ratio": 0.05,
+  "warmup_steps": 0,
+  "log_level": "passive",
+  "log_level_replica": "warning",
+  "log_on_each_node": true,
+  "logging_dir": "/kaggle/tmp/save_checkpoint/v0-20250526-142723/runs",
+  "logging_strategy": "steps",
+  "logging_first_step": true,
+  "logging_steps": 5,
+  "logging_nan_inf_filter": true,
+  "save_strategy": "steps",
+  "save_steps": 640.0,
+  "save_total_limit": 4,
+  "save_safetensors": true,
+  "save_on_each_node": false,
+  "save_only_model": false,
+  "restore_callback_states_from_checkpoint": false,
+  "no_cuda": false,
+  "use_cpu": false,
+  "use_mps_device": false,
+  "jit_mode_eval": false,
+  "use_ipex": false,
+  "bf16": false,
+  "fp16": true,
+  "fp16_opt_level": "O1",
+  "half_precision_backend": "auto",
+  "bf16_full_eval": false,
+  "fp16_full_eval": false,
+  "tf32": null,
+  "local_rank": 0,
+  "tpu_num_cores": null,
+  "tpu_metrics_debug": false,
+  "debug": null,
+  "dataloader_drop_last": false,
+  "eval_steps": 640.0,
+  "dataloader_num_workers": 8,
+  "dataloader_prefetch_factor": null,
+  "past_index": -1,
+  "run_name": "/kaggle/tmp/save_checkpoint/v0-20250526-142723",
+  "disable_tqdm": null,
+  "label_names": null,
+  "load_best_model_at_end": false,
+  "metric_for_best_model": "loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "fsdp": "",
+  "fsdp_min_num_params": 0,
+  "fsdp_config": null,
+  "tp_size": 0,
+  "fsdp_transformer_layer_cls_to_wrap": null,
+  "accelerator_config": {
+    "dispatch_batches": false
+  },
+  "deepspeed": {
+    "fp16": {
+      "enabled": "auto",
+      "loss_scale": 0,
+      "loss_scale_window": 1000,
+      "initial_scale_power": 16,
+      "hysteresis": 2,
+      "min_loss_scale": 1
+    },
+    "bf16": {
+      "enabled": "auto"
+    },
+    "zero_optimization": {
+      "stage": 3,
+      "offload_optimizer": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "none",
+        "pin_memory": true
+      },
+      "overlap_comm": false,
+      "contiguous_gradients": true,
+      "sub_group_size": 1000000000.0,
+      "reduce_bucket_size": "auto",
+      "zero_quantized_weights": false,
+      "zero_quantized_gradients": false,
+      "stage3_prefetch_bucket_size": "auto",
+      "stage3_param_persistence_threshold": "auto",
+      "stage3_max_live_parameters": 1000000000.0,
+      "stage3_max_reuse_distance": 1000000000.0,
+      "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+  },
+  "label_smoothing_factor": 0.0,
+  "optim": "adamw_torch",
+  "optim_args": null,
+  "adafactor": false,
+  "group_by_length": false,
+  "length_column_name": "length",
+  "report_to": [
+    "tensorboard"
+  ],
+  "ddp_find_unused_parameters": null,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataloader_pin_memory": true,
+  "dataloader_persistent_workers": false,
+  "skip_memory_metrics": true,
+  "use_legacy_prediction_loop": false,
+  "push_to_hub": false,
+  "resume_from_checkpoint": null,
+  "hub_model_id": null,
+  "hub_strategy": "every_save",
+  "hub_private_repo": null,
+  "hub_always_push": false,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": null,
+  "include_inputs_for_metrics": false,
+  "include_for_metrics": [],
+  "eval_do_concat_batches": true,
+  "fp16_backend": "auto",
+  "push_to_hub_model_id": null,
+  "push_to_hub_organization": null,
+  "push_to_hub_token": null,
+  "mp_parameters": "",
+  "auto_find_batch_size": false,
+  "full_determinism": false,
+  "torchdynamo": null,
+  "ray_scope": "last",
+  "torch_compile": false,
+  "torch_compile_backend": null,
+  "torch_compile_mode": null,
+  "include_tokens_per_second": false,
+  "include_num_input_tokens_seen": false,
+  "neftune_noise_alpha": null,
+  "optim_target_modules": null,
+  "batch_eval_metrics": false,
+  "eval_on_start": false,
+  "use_liger_kernel": true,
+  "eval_use_gather_object": false,
+  "average_tokens_across_devices": false,
+  "sortish_sampler": false,
+  "predict_with_generate": false,
+  "generation_max_length": null,
+  "generation_num_beams": null,
+  "generation_config": null,
+  "vit_gradient_checkpointing": null,
+  "check_model": true,
+  "acc_strategy": "token",
+  "train_dataloader_shuffle": true,
+  "max_epochs": null,
+  "aligner_lr": null,
+  "vit_lr": null,
+  "optimizer": null,
+  "metric_warmup_step": 0,
+  "fsdp_num": 1,
+  "acc_steps": 1,
+  "eval_use_evalscope": false,
+  "eval_datasets": [],
+  "eval_limit": null,
+  "eval_datasets_args": null,
+  "eval_generation_config": null,
+  "freeze_parameters": [],
+  "freeze_parameters_regex": null,
+  "freeze_parameters_ratio": 0.0,
+  "trainable_parameters": [],
+  "trainable_parameters_regex": null,
+  "freeze_llm": false,
+  "freeze_vit": true,
+  "freeze_aligner": true,
+  "target_modules": [
+    "all-linear"
+  ],
+  "target_regex": null,
+  "modules_to_save": [],
+  "lora_rank": 32,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "lora_dtype": null,
+  "lorap_lr_ratio": null,
+  "use_rslora": false,
+  "use_dora": false,
+  "lora_ga_batch_size": 2,
+  "lora_ga_iters": 2,
+  "lora_ga_max_length": 1024,
+  "lora_ga_direction": "ArB2r",
+  "lora_ga_scale": "stable",
+  "lora_ga_stable_gamma": 16,
+  "init_weights": true,
+  "fourier_n_frequency": 2000,
+  "fourier_scaling": 300.0,
+  "boft_block_size": 4,
+  "boft_block_num": 0,
+  "boft_n_butterfly_factor": 1,
+  "boft_dropout": 0.0,
+  "vera_rank": 256,
+  "vera_projection_prng_key": 0,
+  "vera_dropout": 0.0,
+  "vera_d_initial": 0.1,
+  "adapter_act": "gelu",
+  "adapter_length": 128,
+  "use_galore": false,
+  "galore_target_modules": null,
+  "galore_rank": 128,
+  "galore_update_proj_gap": 50,
+  "galore_scale": 1.0,
+  "galore_proj_type": "std",
+  "galore_optim_per_parameter": false,
+  "galore_with_embedding": false,
+  "galore_quantization": false,
+  "galore_proj_quant": false,
+  "galore_proj_bits": 4,
+  "galore_proj_group_size": 256,
+  "galore_cos_threshold": 0.4,
+  "galore_gamma_proj": 2,
+  "galore_queue_size": 5,
+  "adalora_target_r": 8,
+  "adalora_init_r": 12,
+  "adalora_tinit": 0,
+  "adalora_tfinal": 0,
+  "adalora_deltaT": 1,
+  "adalora_beta1": 0.85,
+  "adalora_beta2": 0.85,
+  "adalora_orth_reg_weight": 0.5,
+  "llamapro_num_new_blocks": 4,
+  "llamapro_num_groups": null,
+  "lisa_activated_layers": 0,
+  "lisa_step_interval": 20,
+  "reft_layer_key": null,
+  "reft_layers": null,
+  "reft_rank": 4,
+  "reft_intervention_type": "LoreftIntervention",
+  "reft_args": null,
+  "swanlab_token": null,
+  "swanlab_project": null,
+  "swanlab_workspace": null,
+  "swanlab_exp_name": null,
+  "swanlab_mode": "cloud",
+  "add_version": true,
+  "resume_only_model": false,
+  "create_checkpoint_symlink": false,
+  "packing": true,
+  "lazy_tokenize": false,
+  "loss_type": null,
+  "metric": null,
+  "zero_hpz_partition_size": null,
+  "rank": 0,
+  "global_world_size": 4,
+  "local_world_size": 4,
+  "model_suffix": "1",
+  "model_info": "ModelInfo(model_type='qwen3', model_dir='/kaggle/input/qwen-3/transformers/32b-awq/1', torch_dtype=torch.float16, max_model_len=40960, quant_method='awq', quant_bits=4, rope_scaling=None, config=None, task_type='causal_lm', num_labels=None)",
+  "model_meta": "ModelMeta(model_type='qwen3', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-0.6B-Base', hf_model_id='Qwen/Qwen3-0.6B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-Base', hf_model_id='Qwen/Qwen3-1.7B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-Base', hf_model_id='Qwen/Qwen3-4B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-Base', hf_model_id='Qwen/Qwen3-8B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-Base', hf_model_id='Qwen/Qwen3-14B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-Base', hf_model_id='Qwen/Qwen3-32B-Base', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B', hf_model_id='Qwen/Qwen3-0.6B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B', hf_model_id='Qwen/Qwen3-1.7B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B', hf_model_id='Qwen/Qwen3-4B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B', hf_model_id='Qwen/Qwen3-8B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B', hf_model_id='Qwen/Qwen3-14B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B', hf_model_id='Qwen/Qwen3-32B', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-0.6B-FP8', hf_model_id='Qwen/Qwen3-0.6B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-1.7B-FP8', hf_model_id='Qwen/Qwen3-1.7B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-FP8', hf_model_id='Qwen/Qwen3-4B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-FP8', hf_model_id='Qwen/Qwen3-8B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-FP8', hf_model_id='Qwen/Qwen3-14B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-FP8', hf_model_id='Qwen/Qwen3-32B-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-AWQ', hf_model_id='Qwen/Qwen3-4B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-8B-AWQ', hf_model_id='Qwen/Qwen3-8B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-14B-AWQ', hf_model_id='Qwen/Qwen3-14B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-32B-AWQ', hf_model_id='Qwen/Qwen3-32B-AWQ', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='swift/Qwen3-32B-AWQ', hf_model_id=None, model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen3', get_function=<function get_model_tokenizer_with_flash_attn at 0x7e1c95bd7880>, model_arch='llama', architectures=['Qwen3ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.51'], tags=[])",
+  "model_dir": "/kaggle/input/qwen-3/transformers/32b-awq/1",
+  "hub": "<class 'swift.hub.hub.MSHub'>",
+  "evaluation_strategy": "steps",
+  "training_args": "Seq2SeqTrainingArguments(output_dir='/kaggle/tmp/save_checkpoint/v0-20250526-142723', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=3e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=3.0, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, lr_scheduler_kwargs=None, warmup_ratio=0.05, warmup_steps=0, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/kaggle/tmp/save_checkpoint/v0-20250526-142723/runs', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=True, logging_steps=5, logging_nan_inf_filter=True, save_strategy=<SaveStrategy.STEPS: 'steps'>, save_steps=640, save_total_limit=4, save_safetensors=True, save_on_each_node=False, save_only_model=False, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, use_ipex=False, bf16=False, fp16=True, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=640, dataloader_num_workers=8, dataloader_prefetch_factor=10, past_index=-1, run_name='/kaggle/tmp/save_checkpoint/v0-20250526-142723', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, tp_size=0, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=<OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=None, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=None, hub_always_push=False, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=True, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, vit_gradient_checkpointing=True, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, optimizer=None, metric_warmup_step=0, fsdp_num=1, acc_steps=1, eval_use_evalscope=False, eval_datasets=[], eval_limit=None, eval_datasets_args=None, eval_generation_config=None, train_type='lora', local_repo_path=None, galore_config=None)"
+}

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff1af3289ba9d861c4fcd687408f06d22ae67e1600a0d6266ef56e4bc5dc0837
+size 16214111450

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:076a40be547cb18e4e0b0df2b44bc7c05f69bb84520ef356b63bf60fb8d9c9b7
+size 805310600

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1001252c25a75572fb298792ea4dd520a5809e1c01c749fb7f57364a2aa3d726
+size 16214111450

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22846cbae9231f77dfe9407a53fd274976d13145cc2cdd04240aa8a94bd00445
+size 805310600

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3e92ce9765953dcbee3f63b855fd247676df9cd68f1ef905ede2ec9280c7247
+size 16214111450

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:087691cbc41e92060c624a117bd188a47333da659fe3937429d4b446d51a36a7
+size 805310600

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be8ba1c740f05aa6f553f0f33c7f952a88aa742ff0cc515240eb53d5d45da857
+size 16214111450

v0-20250526-142723/checkpoint-640/global_step638/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7122b1acdfd418777310e08127e439192845c276acc07bb25b3f129e6ba431c
+size 805310600

v0-20250526-142723/checkpoint-640/latest

@@ -0,0 +1 @@
+global_step638

v0-20250526-142723/checkpoint-640/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ece6b6cdc3888475d045810d85bd19f71338a8dc932c4a9cacba07c4664dd43e
+size 15024

v0-20250526-142723/checkpoint-640/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1b1886e5fae9ba9ac0a7730b8b73207b1f2be2a766e6deeffe8466504eb3518
+size 15024

v0-20250526-142723/checkpoint-640/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9b839f13070e39523c6ed87eba7a2598369f822d84fe8422a7a342b1ed854de
+size 15024

v0-20250526-142723/checkpoint-640/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cacbccf4ecf328a22a2e7528c676c5bd89abb44df99ab9197794c87061ade90a
+size 15024

v0-20250526-142723/checkpoint-640/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f6ac32993f35ca226989dd29f003ef5e8da7dec327c86fb0bf46b0ae034064f
+size 1064

v0-20250526-142723/checkpoint-640/trainer_state.json

@@ -0,0 +1,1195 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.044835868694956,
+  "eval_steps": 640,
+  "global_step": 640,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0032025620496397116,
+      "grad_norm": 1.113493326174143,
+      "learning_rate": 6.382978723404255e-07,
+      "loss": 1.4235601425170898,
+      "memory(GiB)": 21.89,
+      "step": 1,
+      "train_speed(iter/s)": 0.001478
+    },
+    {
+      "epoch": 0.016012810248198558,
+      "grad_norm": 1.2512449636957643,
+      "learning_rate": 3.1914893617021277e-06,
+      "loss": 1.4059971570968628,
+      "memory(GiB)": 21.89,
+      "step": 5,
+      "train_speed(iter/s)": 0.0054
+    },
+    {
+      "epoch": 0.032025620496397116,
+      "grad_norm": 1.1685257009513943,
+      "learning_rate": 6.3829787234042555e-06,
+      "loss": 1.4040182113647461,
+      "memory(GiB)": 21.89,
+      "step": 10,
+      "train_speed(iter/s)": 0.008109
+    },
+    {
+      "epoch": 0.04803843074459568,
+      "grad_norm": 1.0347892126573268,
+      "learning_rate": 9.574468085106385e-06,
+      "loss": 1.2724601745605468,
+      "memory(GiB)": 21.89,
+      "step": 15,
+      "train_speed(iter/s)": 0.009701
+    },
+    {
+      "epoch": 0.06405124099279423,
+      "grad_norm": 0.7170611254380553,
+      "learning_rate": 1.2765957446808511e-05,
+      "loss": 1.1707847595214844,
+      "memory(GiB)": 21.89,
+      "step": 20,
+      "train_speed(iter/s)": 0.010796
+    },
+    {
+      "epoch": 0.08006405124099279,
+      "grad_norm": 0.6048649522124168,
+      "learning_rate": 1.5957446808510637e-05,
+      "loss": 1.0797317504882813,
+      "memory(GiB)": 21.89,
+      "step": 25,
+      "train_speed(iter/s)": 0.011557
+    },
+    {
+      "epoch": 0.09607686148919135,
+      "grad_norm": 0.6938325308343493,
+      "learning_rate": 1.914893617021277e-05,
+      "loss": 0.9897993087768555,
+      "memory(GiB)": 21.89,
+      "step": 30,
+      "train_speed(iter/s)": 0.012125
+    },
+    {
+      "epoch": 0.11208967173738991,
+      "grad_norm": 0.609530150936291,
+      "learning_rate": 2.2340425531914894e-05,
+      "loss": 0.8915694236755372,
+      "memory(GiB)": 21.89,
+      "step": 35,
+      "train_speed(iter/s)": 0.012567
+    },
+    {
+      "epoch": 0.12810248198558846,
+      "grad_norm": 0.6060866850380204,
+      "learning_rate": 2.5531914893617022e-05,
+      "loss": 0.8486058235168457,
+      "memory(GiB)": 21.89,
+      "step": 40,
+      "train_speed(iter/s)": 0.012912
+    },
+    {
+      "epoch": 0.14411529223378702,
+      "grad_norm": 0.5542471477131478,
+      "learning_rate": 2.872340425531915e-05,
+      "loss": 0.7968315124511719,
+      "memory(GiB)": 21.89,
+      "step": 45,
+      "train_speed(iter/s)": 0.013196
+    },
+    {
+      "epoch": 0.16012810248198558,
+      "grad_norm": 0.5328098621829698,
+      "learning_rate": 2.9999157061425123e-05,
+      "loss": 0.7713782310485839,
+      "memory(GiB)": 21.89,
+      "step": 50,
+      "train_speed(iter/s)": 0.013423
+    },
+    {
+      "epoch": 0.17614091273018415,
+      "grad_norm": 0.5124254128934531,
+      "learning_rate": 2.9994006113219365e-05,
+      "loss": 0.7756826400756835,
+      "memory(GiB)": 21.89,
+      "step": 55,
+      "train_speed(iter/s)": 0.013636
+    },
+    {
+      "epoch": 0.1921537229783827,
+      "grad_norm": 0.45894560927350003,
+      "learning_rate": 2.9984174122163244e-05,
+      "loss": 0.7531558990478515,
+      "memory(GiB)": 21.89,
+      "step": 60,
+      "train_speed(iter/s)": 0.013815
+    },
+    {
+      "epoch": 0.20816653322658127,
+      "grad_norm": 0.42532071725821213,
+      "learning_rate": 2.9969664157748082e-05,
+      "loss": 0.720164680480957,
+      "memory(GiB)": 21.89,
+      "step": 65,
+      "train_speed(iter/s)": 0.013962
+    },
+    {
+      "epoch": 0.22417934347477983,
+      "grad_norm": 0.4661145482327595,
+      "learning_rate": 2.995048074990167e-05,
+      "loss": 0.7291761875152588,
+      "memory(GiB)": 21.89,
+      "step": 70,
+      "train_speed(iter/s)": 0.014104
+    },
+    {
+      "epoch": 0.2401921537229784,
+      "grad_norm": 0.5105236274459655,
+      "learning_rate": 2.99266298875741e-05,
+      "loss": 0.7409891128540039,
+      "memory(GiB)": 21.89,
+      "step": 75,
+      "train_speed(iter/s)": 0.014203
+    },
+    {
+      "epoch": 0.2562049639711769,
+      "grad_norm": 0.5866969087739654,
+      "learning_rate": 2.989811901686801e-05,
+      "loss": 0.7214169502258301,
+      "memory(GiB)": 21.89,
+      "step": 80,
+      "train_speed(iter/s)": 0.01431
+    },
+    {
+      "epoch": 0.2722177742193755,
+      "grad_norm": 0.4549841107961782,
+      "learning_rate": 2.986495703871398e-05,
+      "loss": 0.7045151233673096,
+      "memory(GiB)": 21.89,
+      "step": 85,
+      "train_speed(iter/s)": 0.014401
+    },
+    {
+      "epoch": 0.28823058446757405,
+      "grad_norm": 0.4514625540852967,
+      "learning_rate": 2.98271543060917e-05,
+      "loss": 0.6741261005401611,
+      "memory(GiB)": 21.89,
+      "step": 90,
+      "train_speed(iter/s)": 0.014476
+    },
+    {
+      "epoch": 0.3042433947157726,
+      "grad_norm": 0.43958566816038286,
+      "learning_rate": 2.978472262079782e-05,
+      "loss": 0.7089397430419921,
+      "memory(GiB)": 21.89,
+      "step": 95,
+      "train_speed(iter/s)": 0.014542
+    },
+    {
+      "epoch": 0.32025620496397117,
+      "grad_norm": 0.4652252500687268,
+      "learning_rate": 2.973767522976153e-05,
+      "loss": 0.6852937698364258,
+      "memory(GiB)": 21.89,
+      "step": 100,
+      "train_speed(iter/s)": 0.014601
+    },
+    {
+      "epoch": 0.33626901521216973,
+      "grad_norm": 0.4569144232265872,
+      "learning_rate": 2.9686026820908904e-05,
+      "loss": 0.6984831809997558,
+      "memory(GiB)": 21.89,
+      "step": 105,
+      "train_speed(iter/s)": 0.014666
+    },
+    {
+      "epoch": 0.3522818254603683,
+      "grad_norm": 0.49062268332997805,
+      "learning_rate": 2.964140614841668e-05,
+      "loss": 0.6909584522247314,
+      "memory(GiB)": 21.89,
+      "step": 110,
+      "train_speed(iter/s)": 0.014717
+    },
+    {
+      "epoch": 0.36829463570856685,
+      "grad_norm": 0.4599131682455051,
+      "learning_rate": 2.9581517502891373e-05,
+      "loss": 0.7058491706848145,
+      "memory(GiB)": 21.89,
+      "step": 115,
+      "train_speed(iter/s)": 0.014779
+    },
+    {
+      "epoch": 0.3843074459567654,
+      "grad_norm": 0.46586840624727305,
+      "learning_rate": 2.9517076591097877e-05,
+      "loss": 0.6904548645019531,
+      "memory(GiB)": 21.89,
+      "step": 120,
+      "train_speed(iter/s)": 0.014825
+    },
+    {
+      "epoch": 0.400320256204964,
+      "grad_norm": 0.4790148747464638,
+      "learning_rate": 2.9448103531119858e-05,
+      "loss": 0.6842238903045654,
+      "memory(GiB)": 21.89,
+      "step": 125,
+      "train_speed(iter/s)": 0.014876
+    },
+    {
+      "epoch": 0.41633306645316254,
+      "grad_norm": 0.48414224671346245,
+      "learning_rate": 2.9374619855951676e-05,
+      "loss": 0.6715976238250733,
+      "memory(GiB)": 21.89,
+      "step": 130,
+      "train_speed(iter/s)": 0.014914
+    },
+    {
+      "epoch": 0.4323458767013611,
+      "grad_norm": 0.44756883959355714,
+      "learning_rate": 2.929664850677595e-05,
+      "loss": 0.6528284072875976,
+      "memory(GiB)": 21.89,
+      "step": 135,
+      "train_speed(iter/s)": 0.014956
+    },
+    {
+      "epoch": 0.44835868694955966,
+      "grad_norm": 0.4614445331228631,
+      "learning_rate": 2.921421382580142e-05,
+      "loss": 0.6604287624359131,
+      "memory(GiB)": 21.89,
+      "step": 140,
+      "train_speed(iter/s)": 0.014993
+    },
+    {
+      "epoch": 0.4643714971977582,
+      "grad_norm": 0.4552618648718467,
+      "learning_rate": 2.9127341548663437e-05,
+      "loss": 0.663853931427002,
+      "memory(GiB)": 21.89,
+      "step": 145,
+      "train_speed(iter/s)": 0.015021
+    },
+    {
+      "epoch": 0.4803843074459568,
+      "grad_norm": 0.48893303880924177,
+      "learning_rate": 2.9036058796389484e-05,
+      "loss": 0.6526768684387207,
+      "memory(GiB)": 21.89,
+      "step": 150,
+      "train_speed(iter/s)": 0.01506
+    },
+    {
+      "epoch": 0.49639711769415534,
+      "grad_norm": 0.5294944478705774,
+      "learning_rate": 2.8940394066932122e-05,
+      "loss": 0.6768057346343994,
+      "memory(GiB)": 21.89,
+      "step": 155,
+      "train_speed(iter/s)": 0.015082
+    },
+    {
+      "epoch": 0.5124099279423538,
+      "grad_norm": 0.4779511788683914,
+      "learning_rate": 2.8840377226272102e-05,
+      "loss": 0.6269515991210938,
+      "memory(GiB)": 21.89,
+      "step": 160,
+      "train_speed(iter/s)": 0.015104
+    },
+    {
+      "epoch": 0.5284227381905524,
+      "grad_norm": 0.4492505317976044,
+      "learning_rate": 2.873603949909435e-05,
+      "loss": 0.6443400382995605,
+      "memory(GiB)": 21.89,
+      "step": 165,
+      "train_speed(iter/s)": 0.015127
+    },
+    {
+      "epoch": 0.544435548438751,
+      "grad_norm": 0.4997181805996068,
+      "learning_rate": 2.862741345903985e-05,
+      "loss": 0.6411213874816895,
+      "memory(GiB)": 21.89,
+      "step": 170,
+      "train_speed(iter/s)": 0.015159
+    },
+    {
+      "epoch": 0.5604483586869495,
+      "grad_norm": 0.4529198350769972,
+      "learning_rate": 2.8514533018536286e-05,
+      "loss": 0.6319211006164551,
+      "memory(GiB)": 21.89,
+      "step": 175,
+      "train_speed(iter/s)": 0.015177
+    },
+    {
+      "epoch": 0.5764611689351481,
+      "grad_norm": 0.4439771339111341,
+      "learning_rate": 2.839743341821082e-05,
+      "loss": 0.6533251762390136,
+      "memory(GiB)": 21.89,
+      "step": 180,
+      "train_speed(iter/s)": 0.015196
+    },
+    {
+      "epoch": 0.5924739791833467,
+      "grad_norm": 0.4482439997043441,
+      "learning_rate": 2.8276151215888127e-05,
+      "loss": 0.6418941974639892,
+      "memory(GiB)": 21.89,
+      "step": 185,
+      "train_speed(iter/s)": 0.015212
+    },
+    {
+      "epoch": 0.6084867894315452,
+      "grad_norm": 0.4878967638917357,
+      "learning_rate": 2.8150724275177312e-05,
+      "loss": 0.6312544822692872,
+      "memory(GiB)": 21.89,
+      "step": 190,
+      "train_speed(iter/s)": 0.015235
+    },
+    {
+      "epoch": 0.6244995996797438,
+      "grad_norm": 0.48121964708849274,
+      "learning_rate": 2.8021191753651025e-05,
+      "loss": 0.6673158168792724,
+      "memory(GiB)": 21.89,
+      "step": 195,
+      "train_speed(iter/s)": 0.015256
+    },
+    {
+      "epoch": 0.6405124099279423,
+      "grad_norm": 0.46466509624654573,
+      "learning_rate": 2.788759409062078e-05,
+      "loss": 0.6434747695922851,
+      "memory(GiB)": 21.89,
+      "step": 200,
+      "train_speed(iter/s)": 0.015269
+    },
+    {
+      "epoch": 0.6565252201761409,
+      "grad_norm": 0.4673641697899948,
+      "learning_rate": 2.774997299451196e-05,
+      "loss": 0.6431779384613037,
+      "memory(GiB)": 21.89,
+      "step": 205,
+      "train_speed(iter/s)": 0.015292
+    },
+    {
+      "epoch": 0.6725380304243395,
+      "grad_norm": 0.45779788532747007,
+      "learning_rate": 2.760837142984274e-05,
+      "loss": 0.6721176624298095,
+      "memory(GiB)": 21.89,
+      "step": 210,
+      "train_speed(iter/s)": 0.015313
+    },
+    {
+      "epoch": 0.688550840672538,
+      "grad_norm": 0.4618914038595962,
+      "learning_rate": 2.7462833603810768e-05,
+      "loss": 0.6672334671020508,
+      "memory(GiB)": 21.89,
+      "step": 215,
+      "train_speed(iter/s)": 0.015327
+    },
+    {
+      "epoch": 0.7045636509207366,
+      "grad_norm": 0.4764969973944645,
+      "learning_rate": 2.731340495249196e-05,
+      "loss": 0.6564915657043457,
+      "memory(GiB)": 21.89,
+      "step": 220,
+      "train_speed(iter/s)": 0.015342
+    },
+    {
+      "epoch": 0.7205764611689351,
+      "grad_norm": 0.4660822969884439,
+      "learning_rate": 2.7160132126655602e-05,
+      "loss": 0.6348361968994141,
+      "memory(GiB)": 21.89,
+      "step": 225,
+      "train_speed(iter/s)": 0.015365
+    },
+    {
+      "epoch": 0.7365892714171337,
+      "grad_norm": 0.4889253530342789,
+      "learning_rate": 2.700306297720026e-05,
+      "loss": 0.6062727928161621,
+      "memory(GiB)": 21.89,
+      "step": 230,
+      "train_speed(iter/s)": 0.015384
+    },
+    {
+      "epoch": 0.7526020816653323,
+      "grad_norm": 0.5838631024249971,
+      "learning_rate": 2.684224654021498e-05,
+      "loss": 0.6389497756958008,
+      "memory(GiB)": 21.89,
+      "step": 235,
+      "train_speed(iter/s)": 0.0154
+    },
+    {
+      "epoch": 0.7686148919135308,
+      "grad_norm": 0.4607421617033317,
+      "learning_rate": 2.667773302167053e-05,
+      "loss": 0.6525444984436035,
+      "memory(GiB)": 21.89,
+      "step": 240,
+      "train_speed(iter/s)": 0.015413
+    },
+    {
+      "epoch": 0.7846277021617294,
+      "grad_norm": 0.45477587237162076,
+      "learning_rate": 2.6509573781745376e-05,
+      "loss": 0.605835247039795,
+      "memory(GiB)": 21.89,
+      "step": 245,
+      "train_speed(iter/s)": 0.015424
+    },
+    {
+      "epoch": 0.800640512409928,
+      "grad_norm": 0.4730567311540799,
+      "learning_rate": 2.6337821318791303e-05,
+      "loss": 0.6313736438751221,
+      "memory(GiB)": 21.89,
+      "step": 250,
+      "train_speed(iter/s)": 0.015433
+    },
+    {
+      "epoch": 0.8166533226581265,
+      "grad_norm": 0.485561541193875,
+      "learning_rate": 2.6162529252943772e-05,
+      "loss": 0.6605137825012207,
+      "memory(GiB)": 21.89,
+      "step": 255,
+      "train_speed(iter/s)": 0.01545
+    },
+    {
+      "epoch": 0.8326661329063251,
+      "grad_norm": 0.459278946124628,
+      "learning_rate": 2.598375230938198e-05,
+      "loss": 0.6483343124389649,
+      "memory(GiB)": 21.89,
+      "step": 260,
+      "train_speed(iter/s)": 0.015463
+    },
+    {
+      "epoch": 0.8486789431545236,
+      "grad_norm": 0.45947637817712067,
+      "learning_rate": 2.5801546301244004e-05,
+      "loss": 0.6174493789672851,
+      "memory(GiB)": 21.89,
+      "step": 265,
+      "train_speed(iter/s)": 0.015476
+    },
+    {
+      "epoch": 0.8646917534027222,
+      "grad_norm": 0.48423584850188345,
+      "learning_rate": 2.561596811220225e-05,
+      "loss": 0.634972095489502,
+      "memory(GiB)": 21.89,
+      "step": 270,
+      "train_speed(iter/s)": 0.015483
+    },
+    {
+      "epoch": 0.8807045636509208,
+      "grad_norm": 0.48548008083825134,
+      "learning_rate": 2.5427075678704676e-05,
+      "loss": 0.6042546272277832,
+      "memory(GiB)": 21.89,
+      "step": 275,
+      "train_speed(iter/s)": 0.015495
+    },
+    {
+      "epoch": 0.8967173738991193,
+      "grad_norm": 0.45844672211039833,
+      "learning_rate": 2.5234927971887388e-05,
+      "loss": 0.6437081813812255,
+      "memory(GiB)": 21.89,
+      "step": 280,
+      "train_speed(iter/s)": 0.015505
+    },
+    {
+      "epoch": 0.9127301841473179,
+      "grad_norm": 0.49926899360643134,
+      "learning_rate": 2.503958497916419e-05,
+      "loss": 0.6593117713928223,
+      "memory(GiB)": 21.89,
+      "step": 285,
+      "train_speed(iter/s)": 0.015514
+    },
+    {
+      "epoch": 0.9287429943955164,
+      "grad_norm": 0.4768232218990846,
+      "learning_rate": 2.484110768549885e-05,
+      "loss": 0.6409697532653809,
+      "memory(GiB)": 21.89,
+      "step": 290,
+      "train_speed(iter/s)": 0.015526
+    },
+    {
+      "epoch": 0.944755804643715,
+      "grad_norm": 0.5281620311391436,
+      "learning_rate": 2.4639558054365917e-05,
+      "loss": 0.6230706214904785,
+      "memory(GiB)": 21.89,
+      "step": 295,
+      "train_speed(iter/s)": 0.015534
+    },
+    {
+      "epoch": 0.9607686148919136,
+      "grad_norm": 0.60387991785874,
+      "learning_rate": 2.443499900840614e-05,
+      "loss": 0.6483660221099854,
+      "memory(GiB)": 21.89,
+      "step": 300,
+      "train_speed(iter/s)": 0.015545
+    },
+    {
+      "epoch": 0.9767814251401121,
+      "grad_norm": 0.5084362292098538,
+      "learning_rate": 2.422749440978232e-05,
+      "loss": 0.6511590480804443,
+      "memory(GiB)": 21.89,
+      "step": 305,
+      "train_speed(iter/s)": 0.015551
+    },
+    {
+      "epoch": 0.9927942353883107,
+      "grad_norm": 0.497645628004113,
+      "learning_rate": 2.4017109040241948e-05,
+      "loss": 0.6253134727478027,
+      "memory(GiB)": 21.89,
+      "step": 310,
+      "train_speed(iter/s)": 0.01556
+    },
+    {
+      "epoch": 1.0064051240992795,
+      "grad_norm": 0.4950469311671422,
+      "learning_rate": 2.3803908580892694e-05,
+      "loss": 0.6339423179626464,
+      "memory(GiB)": 21.89,
+      "step": 315,
+      "train_speed(iter/s)": 0.015602
+    },
+    {
+      "epoch": 1.022417934347478,
+      "grad_norm": 0.4698835893629837,
+      "learning_rate": 2.358795959169713e-05,
+      "loss": 0.5884588241577149,
+      "memory(GiB)": 21.89,
+      "step": 320,
+      "train_speed(iter/s)": 0.015606
+    },
+    {
+      "epoch": 1.0384307445956766,
+      "grad_norm": 0.5152602884010152,
+      "learning_rate": 2.336932949069314e-05,
+      "loss": 0.594908618927002,
+      "memory(GiB)": 21.89,
+      "step": 325,
+      "train_speed(iter/s)": 0.015611
+    },
+    {
+      "epoch": 1.054443554843875,
+      "grad_norm": 0.498139219735909,
+      "learning_rate": 2.314808653294634e-05,
+      "loss": 0.5890667915344239,
+      "memory(GiB)": 21.89,
+      "step": 330,
+      "train_speed(iter/s)": 0.015616
+    },
+    {
+      "epoch": 1.0704563650920738,
+      "grad_norm": 0.5394223909434638,
+      "learning_rate": 2.2924299789241255e-05,
+      "loss": 0.5816782951354981,
+      "memory(GiB)": 21.89,
+      "step": 335,
+      "train_speed(iter/s)": 0.01562
+    },
+    {
+      "epoch": 1.0864691753402722,
+      "grad_norm": 0.5050953771476231,
+      "learning_rate": 2.2698039124517843e-05,
+      "loss": 0.549614429473877,
+      "memory(GiB)": 21.89,
+      "step": 340,
+      "train_speed(iter/s)": 0.01563
+    },
+    {
+      "epoch": 1.1024819855884709,
+      "grad_norm": 0.5126249224138567,
+      "learning_rate": 2.2469375176060017e-05,
+      "loss": 0.5438549041748046,
+      "memory(GiB)": 21.89,
+      "step": 345,
+      "train_speed(iter/s)": 0.015636
+    },
+    {
+      "epoch": 1.1184947958366693,
+      "grad_norm": 0.5398621051568007,
+      "learning_rate": 2.2238379331443118e-05,
+      "loss": 0.5632871627807617,
+      "memory(GiB)": 21.89,
+      "step": 350,
+      "train_speed(iter/s)": 0.015641
+    },
+    {
+      "epoch": 1.1345076060848678,
+      "grad_norm": 0.5036409732278169,
+      "learning_rate": 2.2005123706247113e-05,
+      "loss": 0.6045978546142579,
+      "memory(GiB)": 21.89,
+      "step": 355,
+      "train_speed(iter/s)": 0.015646
+    },
+    {
+      "epoch": 1.1505204163330665,
+      "grad_norm": 0.5689759899872553,
+      "learning_rate": 2.1769681121542527e-05,
+      "loss": 0.5987513542175293,
+      "memory(GiB)": 21.89,
+      "step": 360,
+      "train_speed(iter/s)": 0.015648
+    },
+    {
+      "epoch": 1.1665332265812651,
+      "grad_norm": 0.531069555558785,
+      "learning_rate": 2.153212508115613e-05,
+      "loss": 0.5959650039672851,
+      "memory(GiB)": 21.89,
+      "step": 365,
+      "train_speed(iter/s)": 0.015657
+    },
+    {
+      "epoch": 1.1825460368294636,
+      "grad_norm": 0.5226537183617093,
+      "learning_rate": 2.1292529748723436e-05,
+      "loss": 0.5860534191131592,
+      "memory(GiB)": 21.89,
+      "step": 370,
+      "train_speed(iter/s)": 0.01566
+    },
+    {
+      "epoch": 1.198558847077662,
+      "grad_norm": 0.5342824123143468,
+      "learning_rate": 2.1050969924535267e-05,
+      "loss": 0.5731143951416016,
+      "memory(GiB)": 21.89,
+      "step": 375,
+      "train_speed(iter/s)": 0.015665
+    },
+    {
+      "epoch": 1.2145716573258607,
+      "grad_norm": 0.5676516907493653,
+      "learning_rate": 2.080752102218553e-05,
+      "loss": 0.5592511177062989,
+      "memory(GiB)": 21.89,
+      "step": 380,
+      "train_speed(iter/s)": 0.015674
+    },
+    {
+      "epoch": 1.2305844675740594,
+      "grad_norm": 0.5615182761459606,
+      "learning_rate": 2.05622590450275e-05,
+      "loss": 0.5661966800689697,
+      "memory(GiB)": 21.89,
+      "step": 385,
+      "train_speed(iter/s)": 0.015681
+    },
+    {
+      "epoch": 1.2465972778222578,
+      "grad_norm": 0.5422157473688541,
+      "learning_rate": 2.0315260562446048e-05,
+      "loss": 0.5789735317230225,
+      "memory(GiB)": 21.89,
+      "step": 390,
+      "train_speed(iter/s)": 0.015684
+    },
+    {
+      "epoch": 1.2626100880704563,
+      "grad_norm": 0.6030245537246277,
+      "learning_rate": 2.0066602685953065e-05,
+      "loss": 0.5753808975219726,
+      "memory(GiB)": 21.89,
+      "step": 395,
+      "train_speed(iter/s)": 0.01569
+    },
+    {
+      "epoch": 1.278622898318655,
+      "grad_norm": 0.6143418775145588,
+      "learning_rate": 1.9816363045113748e-05,
+      "loss": 0.5792715549468994,
+      "memory(GiB)": 21.89,
+      "step": 400,
+      "train_speed(iter/s)": 0.015693
+    },
+    {
+      "epoch": 1.2946357085668536,
+      "grad_norm": 0.5586151801773422,
+      "learning_rate": 1.9564619763311043e-05,
+      "loss": 0.584225845336914,
+      "memory(GiB)": 21.89,
+      "step": 405,
+      "train_speed(iter/s)": 0.015698
+    },
+    {
+      "epoch": 1.310648518815052,
+      "grad_norm": 0.584144416801851,
+      "learning_rate": 1.931145143335601e-05,
+      "loss": 0.5908011436462403,
+      "memory(GiB)": 21.89,
+      "step": 410,
+      "train_speed(iter/s)": 0.015701
+    },
+    {
+      "epoch": 1.3266613290632505,
+      "grad_norm": 0.5965303572137761,
+      "learning_rate": 1.905693709295158e-05,
+      "loss": 0.5673429489135742,
+      "memory(GiB)": 21.89,
+      "step": 415,
+      "train_speed(iter/s)": 0.015705
+    },
+    {
+      "epoch": 1.3426741393114492,
+      "grad_norm": 0.5722923860659136,
+      "learning_rate": 1.880115620001743e-05,
+      "loss": 0.5747467517852783,
+      "memory(GiB)": 21.89,
+      "step": 420,
+      "train_speed(iter/s)": 0.015707
+    },
+    {
+      "epoch": 1.3586869495596476,
+      "grad_norm": 0.5647139487279343,
+      "learning_rate": 1.854418860788369e-05,
+      "loss": 0.5886659622192383,
+      "memory(GiB)": 21.89,
+      "step": 425,
+      "train_speed(iter/s)": 0.015712
+    },
+    {
+      "epoch": 1.3746997598078463,
+      "grad_norm": 0.6032243756929382,
+      "learning_rate": 1.8286114540361192e-05,
+      "loss": 0.5596484184265137,
+      "memory(GiB)": 21.89,
+      "step": 430,
+      "train_speed(iter/s)": 0.015713
+    },
+    {
+      "epoch": 1.3907125700560448,
+      "grad_norm": 0.5798367707195509,
+      "learning_rate": 1.802701456669603e-05,
+      "loss": 0.5662962913513183,
+      "memory(GiB)": 21.89,
+      "step": 435,
+      "train_speed(iter/s)": 0.01572
+    },
+    {
+      "epoch": 1.4067253803042434,
+      "grad_norm": 0.5899381390433364,
+      "learning_rate": 1.776696957641634e-05,
+      "loss": 0.5782527923583984,
+      "memory(GiB)": 21.89,
+      "step": 440,
+      "train_speed(iter/s)": 0.015723
+    },
+    {
+      "epoch": 1.4227381905524419,
+      "grad_norm": 0.5591301892423585,
+      "learning_rate": 1.7506060754079025e-05,
+      "loss": 0.5774744987487793,
+      "memory(GiB)": 21.89,
+      "step": 445,
+      "train_speed(iter/s)": 0.015728
+    },
+    {
+      "epoch": 1.4387510008006406,
+      "grad_norm": 0.5548307822015812,
+      "learning_rate": 1.7244369553924408e-05,
+      "loss": 0.5746015071868896,
+      "memory(GiB)": 21.89,
+      "step": 450,
+      "train_speed(iter/s)": 0.015734
+    },
+    {
+      "epoch": 1.454763811048839,
+      "grad_norm": 0.6162986484237953,
+      "learning_rate": 1.6981977674446692e-05,
+      "loss": 0.5718656063079834,
+      "memory(GiB)": 21.89,
+      "step": 455,
+      "train_speed(iter/s)": 0.015738
+    },
+    {
+      "epoch": 1.4707766212970377,
+      "grad_norm": 0.5999632035162504,
+      "learning_rate": 1.671896703288815e-05,
+      "loss": 0.5838316917419434,
+      "memory(GiB)": 21.89,
+      "step": 460,
+      "train_speed(iter/s)": 0.01574
+    },
+    {
+      "epoch": 1.4867894315452361,
+      "grad_norm": 0.5751934375698643,
+      "learning_rate": 1.6455419739665037e-05,
+      "loss": 0.5416177272796631,
+      "memory(GiB)": 21.89,
+      "step": 465,
+      "train_speed(iter/s)": 0.015742
+    },
+    {
+      "epoch": 1.5028022417934348,
+      "grad_norm": 0.6352358425841226,
+      "learning_rate": 1.6191418072733176e-05,
+      "loss": 0.5688335418701171,
+      "memory(GiB)": 21.89,
+      "step": 470,
+      "train_speed(iter/s)": 0.015747
+    },
+    {
+      "epoch": 1.5188150520416333,
+      "grad_norm": 0.632371720864356,
+      "learning_rate": 1.5927044451901265e-05,
+      "loss": 0.5808904647827149,
+      "memory(GiB)": 21.89,
+      "step": 475,
+      "train_speed(iter/s)": 0.015751
+    },
+    {
+      "epoch": 1.534827862289832,
+      "grad_norm": 0.6298236805463155,
+      "learning_rate": 1.5662381413099885e-05,
+      "loss": 0.5448812007904053,
+      "memory(GiB)": 21.89,
+      "step": 480,
+      "train_speed(iter/s)": 0.015752
+    },
+    {
+      "epoch": 1.5508406725380304,
+      "grad_norm": 0.5876950666906403,
+      "learning_rate": 1.5397511582614238e-05,
+      "loss": 0.5905604839324952,
+      "memory(GiB)": 21.89,
+      "step": 485,
+      "train_speed(iter/s)": 0.015754
+    },
+    {
+      "epoch": 1.5668534827862288,
+      "grad_norm": 0.6384696335208135,
+      "learning_rate": 1.5132517651288716e-05,
+      "loss": 0.5981444835662841,
+      "memory(GiB)": 21.89,
+      "step": 490,
+      "train_speed(iter/s)": 0.015756
+    },
+    {
+      "epoch": 1.5828662930344275,
+      "grad_norm": 0.5982709352921093,
+      "learning_rate": 1.4867482348711284e-05,
+      "loss": 0.5448336124420166,
+      "memory(GiB)": 21.89,
+      "step": 495,
+      "train_speed(iter/s)": 0.015757
+    },
+    {
+      "epoch": 1.5988791032826262,
+      "grad_norm": 0.6299909354063818,
+      "learning_rate": 1.4602488417385766e-05,
+      "loss": 0.5985595703125,
+      "memory(GiB)": 21.89,
+      "step": 500,
+      "train_speed(iter/s)": 0.01576
+    },
+    {
+      "epoch": 1.6148919135308246,
+      "grad_norm": 0.6279367310875671,
+      "learning_rate": 1.4337618586900119e-05,
+      "loss": 0.5594313621520997,
+      "memory(GiB)": 21.89,
+      "step": 505,
+      "train_speed(iter/s)": 0.015763
+    },
+    {
+      "epoch": 1.630904723779023,
+      "grad_norm": 0.6414407834261869,
+      "learning_rate": 1.4072955548098741e-05,
+      "loss": 0.5500194549560546,
+      "memory(GiB)": 21.89,
+      "step": 510,
+      "train_speed(iter/s)": 0.015768
+    },
+    {
+      "epoch": 1.6469175340272217,
+      "grad_norm": 0.6597776832435496,
+      "learning_rate": 1.3808581927266827e-05,
+      "loss": 0.574251365661621,
+      "memory(GiB)": 21.89,
+      "step": 515,
+      "train_speed(iter/s)": 0.01577
+    },
+    {
+      "epoch": 1.6629303442754204,
+      "grad_norm": 0.5981444255387114,
+      "learning_rate": 1.354458026033497e-05,
+      "loss": 0.5687233924865722,
+      "memory(GiB)": 21.89,
+      "step": 520,
+      "train_speed(iter/s)": 0.015773
+    },
+    {
+      "epoch": 1.6789431545236189,
+      "grad_norm": 0.6626788950409892,
+      "learning_rate": 1.3281032967111851e-05,
+      "loss": 0.5612329006195068,
+      "memory(GiB)": 21.89,
+      "step": 525,
+      "train_speed(iter/s)": 0.015776
+    },
+    {
+      "epoch": 1.6949559647718173,
+      "grad_norm": 0.6165757016552958,
+      "learning_rate": 1.3018022325553313e-05,
+      "loss": 0.5564169406890869,
+      "memory(GiB)": 21.89,
+      "step": 530,
+      "train_speed(iter/s)": 0.015778
+    },
+    {
+      "epoch": 1.710968775020016,
+      "grad_norm": 0.6557631786725961,
+      "learning_rate": 1.2755630446075594e-05,
+      "loss": 0.5940975189208985,
+      "memory(GiB)": 21.89,
+      "step": 535,
+      "train_speed(iter/s)": 0.015782
+    },
+    {
+      "epoch": 1.7269815852682147,
+      "grad_norm": 0.6246530913265699,
+      "learning_rate": 1.2493939245920982e-05,
+      "loss": 0.5587568759918213,
+      "memory(GiB)": 21.89,
+      "step": 540,
+      "train_speed(iter/s)": 0.015784
+    },
+    {
+      "epoch": 1.742994395516413,
+      "grad_norm": 0.5688706338790032,
+      "learning_rate": 1.2233030423583662e-05,
+      "loss": 0.52943115234375,
+      "memory(GiB)": 21.89,
+      "step": 545,
+      "train_speed(iter/s)": 0.015785
+    },
+    {
+      "epoch": 1.7590072057646116,
+      "grad_norm": 0.669504073907202,
+      "learning_rate": 1.1972985433303976e-05,
+      "loss": 0.5580001831054687,
+      "memory(GiB)": 21.89,
+      "step": 550,
+      "train_speed(iter/s)": 0.015787
+    },
+    {
+      "epoch": 1.7750200160128102,
+      "grad_norm": 0.6235300104081916,
+      "learning_rate": 1.1713885459638814e-05,
+      "loss": 0.5758135795593262,
+      "memory(GiB)": 21.89,
+      "step": 555,
+      "train_speed(iter/s)": 0.015788
+    },
+    {
+      "epoch": 1.791032826261009,
+      "grad_norm": 0.6352611781580906,
+      "learning_rate": 1.1455811392116308e-05,
+      "loss": 0.5714420795440673,
+      "memory(GiB)": 21.89,
+      "step": 560,
+      "train_speed(iter/s)": 0.015791
+    },
+    {
+      "epoch": 1.8070456365092074,
+      "grad_norm": 0.6806986898527,
+      "learning_rate": 1.1198843799982572e-05,
+      "loss": 0.5534315586090088,
+      "memory(GiB)": 21.89,
+      "step": 565,
+      "train_speed(iter/s)": 0.015793
+    },
+    {
+      "epoch": 1.8230584467574058,
+      "grad_norm": 0.5993577207262759,
+      "learning_rate": 1.0943062907048421e-05,
+      "loss": 0.5411487102508545,
+      "memory(GiB)": 21.89,
+      "step": 570,
+      "train_speed(iter/s)": 0.015794
+    },
+    {
+      "epoch": 1.8390712570056045,
+      "grad_norm": 0.6680590526449376,
+      "learning_rate": 1.0688548566643992e-05,
+      "loss": 0.5467194080352783,
+      "memory(GiB)": 21.89,
+      "step": 575,
+      "train_speed(iter/s)": 0.015797
+    },
+    {
+      "epoch": 1.8550840672538031,
+      "grad_norm": 0.6649716095610546,
+      "learning_rate": 1.0485902421783774e-05,
+      "loss": 0.578249454498291,
+      "memory(GiB)": 21.89,
+      "step": 580,
+      "train_speed(iter/s)": 0.0158
+    },
+    {
+      "epoch": 1.8710968775020016,
+      "grad_norm": 0.6103854609707349,
+      "learning_rate": 1.0233867828334953e-05,
+      "loss": 0.5735835552215576,
+      "memory(GiB)": 21.89,
+      "step": 585,
+      "train_speed(iter/s)": 0.015801
+    },
+    {
+      "epoch": 1.8871096877502,
+      "grad_norm": 0.753072105008473,
+      "learning_rate": 9.98332119405846e-06,
+      "loss": 0.5487753868103027,
+      "memory(GiB)": 21.89,
+      "step": 590,
+      "train_speed(iter/s)": 0.015805
+    },
+    {
+      "epoch": 1.9031224979983987,
+      "grad_norm": 0.6170844249926819,
+      "learning_rate": 9.734340738178836e-06,
+      "loss": 0.5737974166870117,
+      "memory(GiB)": 21.89,
+      "step": 595,
+      "train_speed(iter/s)": 0.015808
+    },
+    {
+      "epoch": 1.9191353082465974,
+      "grad_norm": 0.6451794374019962,
+      "learning_rate": 9.487004190968708e-06,
+      "loss": 0.552049970626831,
+      "memory(GiB)": 21.89,
+      "step": 600,
+      "train_speed(iter/s)": 0.015809
+    },
+    {
+      "epoch": 1.9351481184947958,
+      "grad_norm": 0.6251942278157026,
+      "learning_rate": 9.241388769481819e-06,
+      "loss": 0.5469517707824707,
+      "memory(GiB)": 21.89,
+      "step": 605,
+      "train_speed(iter/s)": 0.015812
+    },
+    {
+      "epoch": 1.9511609287429943,
+      "grad_norm": 0.6198204771719735,
+      "learning_rate": 8.99757115344633e-06,
+      "loss": 0.5650198936462403,
+      "memory(GiB)": 21.89,
+      "step": 610,
+      "train_speed(iter/s)": 0.015812
+    },
+    {
+      "epoch": 1.967173738991193,
+      "grad_norm": 0.6356161184262821,
+      "learning_rate": 8.755627461325817e-06,
+      "loss": 0.5307876586914062,
+      "memory(GiB)": 21.89,
+      "step": 615,
+      "train_speed(iter/s)": 0.015813
+    },
+    {
+      "epoch": 1.9831865492393916,
+      "grad_norm": 0.6601174519481827,
+      "learning_rate": 8.515633226555564e-06,
+      "loss": 0.5552098274230957,
+      "memory(GiB)": 21.89,
+      "step": 620,
+      "train_speed(iter/s)": 0.015815
+    },
+    {
+      "epoch": 1.99919935948759,
+      "grad_norm": 0.6493635460743743,
+      "learning_rate": 8.277663373961396e-06,
+      "loss": 0.5575791358947754,
+      "memory(GiB)": 21.89,
+      "step": 625,
+      "train_speed(iter/s)": 0.015818
+    },
+    {
+      "epoch": 2.012810248198559,
+      "grad_norm": 0.6130205867475395,
+      "learning_rate": 8.04179219636866e-06,
+      "loss": 0.5210562705993652,
+      "memory(GiB)": 21.89,
+      "step": 630,
+      "train_speed(iter/s)": 0.015839
+    },
+    {
+      "epoch": 2.0288230584467573,
+      "grad_norm": 0.6169564813848567,
+      "learning_rate": 7.808093331408354e-06,
+      "loss": 0.5348440647125244,
+      "memory(GiB)": 21.89,
+      "step": 635,
+      "train_speed(iter/s)": 0.015839
+    },
+    {
+      "epoch": 2.044835868694956,
+      "grad_norm": 0.6502484575021102,
+      "learning_rate": 7.5766397385279204e-06,
+      "loss": 0.4850314140319824,
+      "memory(GiB)": 21.89,
+      "step": 640,
+      "train_speed(iter/s)": 0.015839
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 936,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 640,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 240166800818176.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

v0-20250526-142723/checkpoint-640/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858c61ca7c16487e1f1d03f40e0f57220ac2d4a688d05729201fe8a4acd1052b
+size 8312

v0-20250526-142723/checkpoint-640/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

v0-20250526-142723/logging.jsonl

@@ -0,0 +1,130 @@
+{"loss": 1.42356014, "grad_norm": 1.11349333, "learning_rate": 6.4e-07, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.001478, "epoch": 0.00320256, "global_step/max_steps": "1/936", "percentage": "0.11%", "elapsed_time": "6m 51s", "remaining_time": "4d 10h 56m 8s"}
+{"loss": 1.40599716, "grad_norm": 1.25124496, "learning_rate": 3.19e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.0054, "epoch": 0.01601281, "global_step/max_steps": "5/936", "percentage": "0.53%", "elapsed_time": "11m 1s", "remaining_time": "1d 10h 11m 30s"}
+{"loss": 1.40401821, "grad_norm": 1.1685257, "learning_rate": 6.38e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.008109, "epoch": 0.03202562, "global_step/max_steps": "10/936", "percentage": "1.07%", "elapsed_time": "16m 8s", "remaining_time": "1d 0h 54m 23s"}
+{"loss": 1.27246017, "grad_norm": 1.03478921, "learning_rate": 9.57e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.009701, "epoch": 0.04803843, "global_step/max_steps": "15/936", "percentage": "1.60%", "elapsed_time": "21m 21s", "remaining_time": "21h 51m 19s"}
+{"loss": 1.17078476, "grad_norm": 0.71706113, "learning_rate": 1.277e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.010796, "epoch": 0.06405124, "global_step/max_steps": "20/936", "percentage": "2.14%", "elapsed_time": "26m 27s", "remaining_time": "20h 11m 57s"}
+{"loss": 1.07973175, "grad_norm": 0.60486495, "learning_rate": 1.596e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.011557, "epoch": 0.08006405, "global_step/max_steps": "25/936", "percentage": "2.67%", "elapsed_time": "31m 38s", "remaining_time": "19h 12m 53s"}
+{"loss": 0.98979931, "grad_norm": 0.69383253, "learning_rate": 1.915e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.012125, "epoch": 0.09607686, "global_step/max_steps": "30/936", "percentage": "3.21%", "elapsed_time": "36m 49s", "remaining_time": "18h 32m 0s"}
+{"loss": 0.89156942, "grad_norm": 0.60953015, "learning_rate": 2.234e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.012567, "epoch": 0.11208967, "global_step/max_steps": "35/936", "percentage": "3.74%", "elapsed_time": "42m 0s", "remaining_time": "18h 1m 15s"}
+{"loss": 0.84860582, "grad_norm": 0.60608669, "learning_rate": 2.553e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.012912, "epoch": 0.12810248, "global_step/max_steps": "40/936", "percentage": "4.27%", "elapsed_time": "47m 12s", "remaining_time": "17h 37m 38s"}
+{"loss": 0.79683151, "grad_norm": 0.55424715, "learning_rate": 2.872e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.013196, "epoch": 0.14411529, "global_step/max_steps": "45/936", "percentage": "4.81%", "elapsed_time": "52m 25s", "remaining_time": "17h 17m 58s"}
+{"loss": 0.77137823, "grad_norm": 0.53280986, "learning_rate": 3e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.013423, "epoch": 0.1601281, "global_step/max_steps": "50/936", "percentage": "5.34%", "elapsed_time": "57m 39s", "remaining_time": "17h 1m 50s"}
+{"loss": 0.77568264, "grad_norm": 0.51242541, "learning_rate": 2.999e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.013636, "epoch": 0.17614091, "global_step/max_steps": "55/936", "percentage": "5.88%", "elapsed_time": "1h 2m 48s", "remaining_time": "16h 46m 5s"}
+{"loss": 0.7531559, "grad_norm": 0.45894561, "learning_rate": 2.998e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.013815, "epoch": 0.19215372, "global_step/max_steps": "60/936", "percentage": "6.41%", "elapsed_time": "1h 7m 58s", "remaining_time": "16h 32m 23s"}
+{"loss": 0.72016468, "grad_norm": 0.42532072, "learning_rate": 2.997e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.013962, "epoch": 0.20816653, "global_step/max_steps": "65/936", "percentage": "6.94%", "elapsed_time": "1h 13m 10s", "remaining_time": "16h 20m 32s"}
+{"loss": 0.72917619, "grad_norm": 0.46611455, "learning_rate": 2.995e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014104, "epoch": 0.22417934, "global_step/max_steps": "70/936", "percentage": "7.48%", "elapsed_time": "1h 18m 18s", "remaining_time": "16h 8m 44s"}
+{"loss": 0.74098911, "grad_norm": 0.51052363, "learning_rate": 2.993e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014203, "epoch": 0.24019215, "global_step/max_steps": "75/936", "percentage": "8.01%", "elapsed_time": "1h 23m 35s", "remaining_time": "15h 59m 40s"}
+{"loss": 0.72141695, "grad_norm": 0.58669691, "learning_rate": 2.99e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01431, "epoch": 0.25620496, "global_step/max_steps": "80/936", "percentage": "8.55%", "elapsed_time": "1h 28m 45s", "remaining_time": "15h 49m 45s"}
+{"loss": 0.70451512, "grad_norm": 0.45498411, "learning_rate": 2.986e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014401, "epoch": 0.27221777, "global_step/max_steps": "85/936", "percentage": "9.08%", "elapsed_time": "1h 33m 57s", "remaining_time": "15h 40m 40s"}
+{"loss": 0.6741261, "grad_norm": 0.45146255, "learning_rate": 2.983e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014476, "epoch": 0.28823058, "global_step/max_steps": "90/936", "percentage": "9.62%", "elapsed_time": "1h 39m 12s", "remaining_time": "15h 32m 31s"}
+{"loss": 0.70893974, "grad_norm": 0.43958567, "learning_rate": 2.978e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014542, "epoch": 0.30424339, "global_step/max_steps": "95/936", "percentage": "10.15%", "elapsed_time": "1h 44m 27s", "remaining_time": "15h 24m 46s"}
+{"loss": 0.68529377, "grad_norm": 0.46522525, "learning_rate": 2.974e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014601, "epoch": 0.3202562, "global_step/max_steps": "100/936", "percentage": "10.68%", "elapsed_time": "1h 49m 43s", "remaining_time": "15h 17m 22s"}
+{"loss": 0.69848318, "grad_norm": 0.45691442, "learning_rate": 2.969e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014666, "epoch": 0.33626902, "global_step/max_steps": "105/936", "percentage": "11.22%", "elapsed_time": "1h 54m 54s", "remaining_time": "15h 9m 24s"}
+{"loss": 0.69095845, "grad_norm": 0.49062268, "learning_rate": 2.964e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014717, "epoch": 0.35228183, "global_step/max_steps": "110/936", "percentage": "11.75%", "elapsed_time": "2h 0m 9s", "remaining_time": "15h 2m 15s"}
+{"loss": 0.70584917, "grad_norm": 0.45991317, "learning_rate": 2.958e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014779, "epoch": 0.36829464, "global_step/max_steps": "115/936", "percentage": "12.29%", "elapsed_time": "2h 5m 16s", "remaining_time": "14h 54m 21s"}
+{"loss": 0.69045486, "grad_norm": 0.46586841, "learning_rate": 2.952e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014825, "epoch": 0.38430745, "global_step/max_steps": "120/936", "percentage": "12.82%", "elapsed_time": "2h 10m 29s", "remaining_time": "14h 47m 20s"}
+{"loss": 0.68422389, "grad_norm": 0.47901487, "learning_rate": 2.945e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014876, "epoch": 0.40032026, "global_step/max_steps": "125/936", "percentage": "13.35%", "elapsed_time": "2h 15m 38s", "remaining_time": "14h 40m 0s"}
+{"loss": 0.67159762, "grad_norm": 0.48414225, "learning_rate": 2.937e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014914, "epoch": 0.41633307, "global_step/max_steps": "130/936", "percentage": "13.89%", "elapsed_time": "2h 20m 51s", "remaining_time": "14h 33m 19s"}
+{"loss": 0.65282841, "grad_norm": 0.44756884, "learning_rate": 2.93e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014956, "epoch": 0.43234588, "global_step/max_steps": "135/936", "percentage": "14.42%", "elapsed_time": "2h 26m 1s", "remaining_time": "14h 26m 25s"}
+{"loss": 0.66042876, "grad_norm": 0.46144453, "learning_rate": 2.921e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.014993, "epoch": 0.44835869, "global_step/max_steps": "140/936", "percentage": "14.96%", "elapsed_time": "2h 31m 12s", "remaining_time": "14h 19m 45s"}
+{"loss": 0.66385393, "grad_norm": 0.45526186, "learning_rate": 2.913e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015021, "epoch": 0.4643715, "global_step/max_steps": "145/936", "percentage": "15.49%", "elapsed_time": "2h 36m 28s", "remaining_time": "14h 13m 35s"}
+{"loss": 0.65267687, "grad_norm": 0.48893304, "learning_rate": 2.904e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01506, "epoch": 0.48038431, "global_step/max_steps": "150/936", "percentage": "16.03%", "elapsed_time": "2h 41m 35s", "remaining_time": "14h 6m 42s"}
+{"loss": 0.67680573, "grad_norm": 0.52949445, "learning_rate": 2.894e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015082, "epoch": 0.49639712, "global_step/max_steps": "155/936", "percentage": "16.56%", "elapsed_time": "2h 46m 52s", "remaining_time": "14h 0m 47s"}
+{"loss": 0.6269516, "grad_norm": 0.47795118, "learning_rate": 2.884e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015104, "epoch": 0.51240993, "global_step/max_steps": "160/936", "percentage": "17.09%", "elapsed_time": "2h 52m 8s", "remaining_time": "13h 54m 52s"}
+{"loss": 0.64434004, "grad_norm": 0.44925053, "learning_rate": 2.874e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015127, "epoch": 0.52842274, "global_step/max_steps": "165/936", "percentage": "17.63%", "elapsed_time": "2h 57m 22s", "remaining_time": "13h 48m 49s"}
+{"loss": 0.64112139, "grad_norm": 0.49971818, "learning_rate": 2.863e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015159, "epoch": 0.54443555, "global_step/max_steps": "170/936", "percentage": "18.16%", "elapsed_time": "3h 2m 29s", "remaining_time": "13h 42m 17s"}
+{"loss": 0.6319211, "grad_norm": 0.45291984, "learning_rate": 2.851e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015177, "epoch": 0.56044836, "global_step/max_steps": "175/936", "percentage": "18.70%", "elapsed_time": "3h 7m 45s", "remaining_time": "13h 36m 30s"}
+{"loss": 0.65332518, "grad_norm": 0.44397713, "learning_rate": 2.84e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015196, "epoch": 0.57646117, "global_step/max_steps": "180/936", "percentage": "19.23%", "elapsed_time": "3h 13m 0s", "remaining_time": "13h 30m 38s"}
+{"loss": 0.6418942, "grad_norm": 0.448244, "learning_rate": 2.828e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015212, "epoch": 0.59247398, "global_step/max_steps": "185/936", "percentage": "19.76%", "elapsed_time": "3h 18m 16s", "remaining_time": "13h 24m 54s"}
+{"loss": 0.63125448, "grad_norm": 0.48789676, "learning_rate": 2.815e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015235, "epoch": 0.60848679, "global_step/max_steps": "190/936", "percentage": "20.30%", "elapsed_time": "3h 23m 26s", "remaining_time": "13h 18m 46s"}
+{"loss": 0.66731582, "grad_norm": 0.48121965, "learning_rate": 2.802e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015256, "epoch": 0.6244996, "global_step/max_steps": "195/936", "percentage": "20.83%", "elapsed_time": "3h 28m 36s", "remaining_time": "13h 12m 43s"}
+{"loss": 0.64347477, "grad_norm": 0.4646651, "learning_rate": 2.789e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015269, "epoch": 0.64051241, "global_step/max_steps": "200/936", "percentage": "21.37%", "elapsed_time": "3h 33m 53s", "remaining_time": "13h 7m 7s"}
+{"loss": 0.64317794, "grad_norm": 0.46736417, "learning_rate": 2.775e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015292, "epoch": 0.65652522, "global_step/max_steps": "205/936", "percentage": "21.90%", "elapsed_time": "3h 39m 0s", "remaining_time": "13h 0m 58s"}
+{"loss": 0.67211766, "grad_norm": 0.45779789, "learning_rate": 2.761e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015313, "epoch": 0.67253803, "global_step/max_steps": "210/936", "percentage": "22.44%", "elapsed_time": "3h 44m 8s", "remaining_time": "12h 54m 54s"}
+{"loss": 0.66723347, "grad_norm": 0.4618914, "learning_rate": 2.746e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015327, "epoch": 0.68855084, "global_step/max_steps": "215/936", "percentage": "22.97%", "elapsed_time": "3h 49m 23s", "remaining_time": "12h 49m 14s"}
+{"loss": 0.65649157, "grad_norm": 0.476497, "learning_rate": 2.731e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015342, "epoch": 0.70456365, "global_step/max_steps": "220/936", "percentage": "23.50%", "elapsed_time": "3h 54m 34s", "remaining_time": "12h 43m 26s"}
+{"loss": 0.6348362, "grad_norm": 0.4660823, "learning_rate": 2.716e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015365, "epoch": 0.72057646, "global_step/max_steps": "225/936", "percentage": "24.04%", "elapsed_time": "3h 59m 38s", "remaining_time": "12h 37m 17s"}
+{"loss": 0.60627279, "grad_norm": 0.48892535, "learning_rate": 2.7e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015384, "epoch": 0.73658927, "global_step/max_steps": "230/936", "percentage": "24.57%", "elapsed_time": "4h 4m 45s", "remaining_time": "12h 31m 18s"}
+{"loss": 0.63894978, "grad_norm": 0.5838631, "learning_rate": 2.684e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.0154, "epoch": 0.75260208, "global_step/max_steps": "235/936", "percentage": "25.11%", "elapsed_time": "4h 9m 55s", "remaining_time": "12h 25m 29s"}
+{"loss": 0.6525445, "grad_norm": 0.46074216, "learning_rate": 2.668e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015413, "epoch": 0.76861489, "global_step/max_steps": "240/936", "percentage": "25.64%", "elapsed_time": "4h 15m 6s", "remaining_time": "12h 19m 49s"}
+{"loss": 0.60583525, "grad_norm": 0.45477587, "learning_rate": 2.651e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015424, "epoch": 0.7846277, "global_step/max_steps": "245/936", "percentage": "26.18%", "elapsed_time": "4h 20m 19s", "remaining_time": "12h 14m 14s"}
+{"loss": 0.63137364, "grad_norm": 0.47305673, "learning_rate": 2.634e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015433, "epoch": 0.80064051, "global_step/max_steps": "250/936", "percentage": "26.71%", "elapsed_time": "4h 25m 34s", "remaining_time": "12h 8m 44s"}
+{"loss": 0.66051378, "grad_norm": 0.48556154, "learning_rate": 2.616e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01545, "epoch": 0.81665332, "global_step/max_steps": "255/936", "percentage": "27.24%", "elapsed_time": "4h 30m 39s", "remaining_time": "12h 2m 49s"}
+{"loss": 0.64833431, "grad_norm": 0.45927895, "learning_rate": 2.598e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015463, "epoch": 0.83266613, "global_step/max_steps": "260/936", "percentage": "27.78%", "elapsed_time": "4h 35m 49s", "remaining_time": "11h 57m 7s"}
+{"loss": 0.61744938, "grad_norm": 0.45947638, "learning_rate": 2.58e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015476, "epoch": 0.84867894, "global_step/max_steps": "265/936", "percentage": "28.31%", "elapsed_time": "4h 40m 58s", "remaining_time": "11h 51m 27s"}
+{"loss": 0.6349721, "grad_norm": 0.48423585, "learning_rate": 2.562e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015483, "epoch": 0.86469175, "global_step/max_steps": "270/936", "percentage": "28.85%", "elapsed_time": "4h 46m 13s", "remaining_time": "11h 46m 1s"}
+{"loss": 0.60425463, "grad_norm": 0.48548008, "learning_rate": 2.543e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015495, "epoch": 0.88070456, "global_step/max_steps": "275/936", "percentage": "29.38%", "elapsed_time": "4h 51m 23s", "remaining_time": "11h 40m 23s"}
+{"loss": 0.64370818, "grad_norm": 0.45844672, "learning_rate": 2.523e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015505, "epoch": 0.89671737, "global_step/max_steps": "280/936", "percentage": "29.91%", "elapsed_time": "4h 56m 34s", "remaining_time": "11h 34m 49s"}
+{"loss": 0.65931177, "grad_norm": 0.49926899, "learning_rate": 2.504e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015514, "epoch": 0.91273018, "global_step/max_steps": "285/936", "percentage": "30.45%", "elapsed_time": "5h 1m 45s", "remaining_time": "11h 29m 16s"}
+{"loss": 0.64096975, "grad_norm": 0.47682322, "learning_rate": 2.484e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015526, "epoch": 0.92874299, "global_step/max_steps": "290/936", "percentage": "30.98%", "elapsed_time": "5h 6m 53s", "remaining_time": "11h 23m 38s"}
+{"loss": 0.62307062, "grad_norm": 0.52816203, "learning_rate": 2.464e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015534, "epoch": 0.9447558, "global_step/max_steps": "295/936", "percentage": "31.52%", "elapsed_time": "5h 12m 6s", "remaining_time": "11h 18m 9s"}
+{"loss": 0.64836602, "grad_norm": 0.60387992, "learning_rate": 2.443e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015545, "epoch": 0.96076861, "global_step/max_steps": "300/936", "percentage": "32.05%", "elapsed_time": "5h 17m 14s", "remaining_time": "11h 12m 32s"}
+{"loss": 0.65115905, "grad_norm": 0.50843623, "learning_rate": 2.423e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015551, "epoch": 0.97678143, "global_step/max_steps": "305/936", "percentage": "32.59%", "elapsed_time": "5h 22m 28s", "remaining_time": "11h 7m 8s"}
+{"loss": 0.62531347, "grad_norm": 0.49764563, "learning_rate": 2.402e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01556, "epoch": 0.99279424, "global_step/max_steps": "310/936", "percentage": "33.12%", "elapsed_time": "5h 27m 38s", "remaining_time": "11h 1m 37s"}
+{"loss": 0.63394232, "grad_norm": 0.49504693, "learning_rate": 2.38e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015602, "epoch": 1.00640512, "global_step/max_steps": "315/936", "percentage": "33.65%", "elapsed_time": "5h 32m 4s", "remaining_time": "10h 54m 40s"}
+{"loss": 0.58845882, "grad_norm": 0.46988359, "learning_rate": 2.359e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015606, "epoch": 1.02241793, "global_step/max_steps": "320/936", "percentage": "34.19%", "elapsed_time": "5h 37m 19s", "remaining_time": "10h 49m 21s"}
+{"loss": 0.59490862, "grad_norm": 0.51526029, "learning_rate": 2.337e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015611, "epoch": 1.03843074, "global_step/max_steps": "325/936", "percentage": "34.72%", "elapsed_time": "5h 42m 33s", "remaining_time": "10h 44m 0s"}
+{"loss": 0.58906679, "grad_norm": 0.49813922, "learning_rate": 2.315e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015616, "epoch": 1.05444355, "global_step/max_steps": "330/936", "percentage": "35.26%", "elapsed_time": "5h 47m 47s", "remaining_time": "10h 38m 40s"}
+{"loss": 0.5816783, "grad_norm": 0.53942239, "learning_rate": 2.292e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01562, "epoch": 1.07045637, "global_step/max_steps": "335/936", "percentage": "35.79%", "elapsed_time": "5h 53m 2s", "remaining_time": "10h 33m 21s"}
+{"loss": 0.54961443, "grad_norm": 0.50509538, "learning_rate": 2.27e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01563, "epoch": 1.08646918, "global_step/max_steps": "340/936", "percentage": "36.32%", "elapsed_time": "5h 58m 8s", "remaining_time": "10h 27m 48s"}
+{"loss": 0.5438549, "grad_norm": 0.51262492, "learning_rate": 2.247e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015636, "epoch": 1.10248199, "global_step/max_steps": "345/936", "percentage": "36.86%", "elapsed_time": "6h 3m 19s", "remaining_time": "10h 22m 23s"}
+{"loss": 0.56328716, "grad_norm": 0.53986211, "learning_rate": 2.224e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015641, "epoch": 1.1184948, "global_step/max_steps": "350/936", "percentage": "37.39%", "elapsed_time": "6h 8m 32s", "remaining_time": "10h 17m 3s"}
+{"loss": 0.60459785, "grad_norm": 0.50364097, "learning_rate": 2.201e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015646, "epoch": 1.13450761, "global_step/max_steps": "355/936", "percentage": "37.93%", "elapsed_time": "6h 13m 44s", "remaining_time": "10h 11m 39s"}
+{"loss": 0.59875135, "grad_norm": 0.56897599, "learning_rate": 2.177e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015648, "epoch": 1.15052042, "global_step/max_steps": "360/936", "percentage": "38.46%", "elapsed_time": "6h 19m 0s", "remaining_time": "10h 6m 25s"}
+{"loss": 0.595965, "grad_norm": 0.53106956, "learning_rate": 2.153e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015657, "epoch": 1.16653323, "global_step/max_steps": "365/936", "percentage": "39.00%", "elapsed_time": "6h 24m 7s", "remaining_time": "10h 0m 55s"}
+{"loss": 0.58605342, "grad_norm": 0.52265372, "learning_rate": 2.129e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01566, "epoch": 1.18254604, "global_step/max_steps": "370/936", "percentage": "39.53%", "elapsed_time": "6h 29m 22s", "remaining_time": "9h 55m 38s"}
+{"loss": 0.5731144, "grad_norm": 0.53428241, "learning_rate": 2.105e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015665, "epoch": 1.19855885, "global_step/max_steps": "375/936", "percentage": "40.06%", "elapsed_time": "6h 34m 33s", "remaining_time": "9h 50m 14s"}
+{"loss": 0.55925112, "grad_norm": 0.56765169, "learning_rate": 2.081e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015674, "epoch": 1.21457166, "global_step/max_steps": "380/936", "percentage": "40.60%", "elapsed_time": "6h 39m 39s", "remaining_time": "9h 44m 46s"}
+{"loss": 0.56619668, "grad_norm": 0.56151828, "learning_rate": 2.056e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015681, "epoch": 1.23058447, "global_step/max_steps": "385/936", "percentage": "41.13%", "elapsed_time": "6h 44m 46s", "remaining_time": "9h 39m 18s"}
+{"loss": 0.57897353, "grad_norm": 0.54221575, "learning_rate": 2.032e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015684, "epoch": 1.24659728, "global_step/max_steps": "390/936", "percentage": "41.67%", "elapsed_time": "6h 50m 1s", "remaining_time": "9h 34m 1s"}
+{"loss": 0.5753809, "grad_norm": 0.60302455, "learning_rate": 2.007e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01569, "epoch": 1.26261009, "global_step/max_steps": "395/936", "percentage": "42.20%", "elapsed_time": "6h 55m 9s", "remaining_time": "9h 28m 37s"}
+{"loss": 0.57927155, "grad_norm": 0.61434188, "learning_rate": 1.982e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015693, "epoch": 1.2786229, "global_step/max_steps": "400/936", "percentage": "42.74%", "elapsed_time": "7h 0m 24s", "remaining_time": "9h 23m 20s"}
+{"loss": 0.58422585, "grad_norm": 0.55861518, "learning_rate": 1.956e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015698, "epoch": 1.29463571, "global_step/max_steps": "405/936", "percentage": "43.27%", "elapsed_time": "7h 5m 34s", "remaining_time": "9h 17m 58s"}
+{"loss": 0.59080114, "grad_norm": 0.58414442, "learning_rate": 1.931e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015701, "epoch": 1.31064852, "global_step/max_steps": "410/936", "percentage": "43.80%", "elapsed_time": "7h 10m 48s", "remaining_time": "9h 12m 42s"}
+{"loss": 0.56734295, "grad_norm": 0.59653036, "learning_rate": 1.906e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015705, "epoch": 1.32666133, "global_step/max_steps": "415/936", "percentage": "44.34%", "elapsed_time": "7h 15m 59s", "remaining_time": "9h 7m 21s"}
+{"loss": 0.57474675, "grad_norm": 0.57229239, "learning_rate": 1.88e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015707, "epoch": 1.34267414, "global_step/max_steps": "420/936", "percentage": "44.87%", "elapsed_time": "7h 21m 15s", "remaining_time": "9h 2m 6s"}
+{"loss": 0.58866596, "grad_norm": 0.56471395, "learning_rate": 1.854e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015712, "epoch": 1.35868695, "global_step/max_steps": "425/936", "percentage": "45.41%", "elapsed_time": "7h 26m 23s", "remaining_time": "8h 56m 43s"}
+{"loss": 0.55964842, "grad_norm": 0.60322438, "learning_rate": 1.829e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015713, "epoch": 1.37469976, "global_step/max_steps": "430/936", "percentage": "45.94%", "elapsed_time": "7h 31m 41s", "remaining_time": "8h 51m 31s"}
+{"loss": 0.56629629, "grad_norm": 0.57983677, "learning_rate": 1.803e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01572, "epoch": 1.39071257, "global_step/max_steps": "435/936", "percentage": "46.47%", "elapsed_time": "7h 36m 47s", "remaining_time": "8h 46m 5s"}
+{"loss": 0.57825279, "grad_norm": 0.58993814, "learning_rate": 1.777e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015723, "epoch": 1.40672538, "global_step/max_steps": "440/936", "percentage": "47.01%", "elapsed_time": "7h 41m 58s", "remaining_time": "8h 40m 46s"}
+{"loss": 0.5774745, "grad_norm": 0.55913019, "learning_rate": 1.751e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015728, "epoch": 1.42273819, "global_step/max_steps": "445/936", "percentage": "47.54%", "elapsed_time": "7h 47m 7s", "remaining_time": "8h 35m 25s"}
+{"loss": 0.57460151, "grad_norm": 0.55483078, "learning_rate": 1.724e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015734, "epoch": 1.438751, "global_step/max_steps": "450/936", "percentage": "48.08%", "elapsed_time": "7h 52m 15s", "remaining_time": "8h 30m 2s"}
+{"loss": 0.57186561, "grad_norm": 0.61629865, "learning_rate": 1.698e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015738, "epoch": 1.45476381, "global_step/max_steps": "455/936", "percentage": "48.61%", "elapsed_time": "7h 57m 26s", "remaining_time": "8h 24m 43s"}
+{"loss": 0.58383169, "grad_norm": 0.5999632, "learning_rate": 1.672e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01574, "epoch": 1.47077662, "global_step/max_steps": "460/936", "percentage": "49.15%", "elapsed_time": "8h 2m 39s", "remaining_time": "8h 19m 26s"}
+{"loss": 0.54161773, "grad_norm": 0.57519344, "learning_rate": 1.646e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015742, "epoch": 1.48678943, "global_step/max_steps": "465/936", "percentage": "49.68%", "elapsed_time": "8h 7m 54s", "remaining_time": "8h 14m 11s"}
+{"loss": 0.56883354, "grad_norm": 0.63523584, "learning_rate": 1.619e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015747, "epoch": 1.50280224, "global_step/max_steps": "470/936", "percentage": "50.21%", "elapsed_time": "8h 13m 2s", "remaining_time": "8h 8m 50s"}
+{"loss": 0.58089046, "grad_norm": 0.63237172, "learning_rate": 1.593e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015751, "epoch": 1.51881505, "global_step/max_steps": "475/936", "percentage": "50.75%", "elapsed_time": "8h 18m 11s", "remaining_time": "8h 3m 30s"}
+{"loss": 0.5448812, "grad_norm": 0.62982368, "learning_rate": 1.566e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015752, "epoch": 1.53482786, "global_step/max_steps": "480/936", "percentage": "51.28%", "elapsed_time": "8h 23m 27s", "remaining_time": "7h 58m 17s"}
+{"loss": 0.59056048, "grad_norm": 0.58769507, "learning_rate": 1.54e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015754, "epoch": 1.55084067, "global_step/max_steps": "485/936", "percentage": "51.82%", "elapsed_time": "8h 28m 41s", "remaining_time": "7h 53m 1s"}
+{"loss": 0.59814448, "grad_norm": 0.63846963, "learning_rate": 1.513e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015756, "epoch": 1.56685348, "global_step/max_steps": "490/936", "percentage": "52.35%", "elapsed_time": "8h 33m 53s", "remaining_time": "7h 47m 44s"}
+{"loss": 0.54483361, "grad_norm": 0.59827094, "learning_rate": 1.487e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015757, "epoch": 1.58286629, "global_step/max_steps": "495/936", "percentage": "52.88%", "elapsed_time": "8h 39m 10s", "remaining_time": "7h 42m 31s"}
+{"loss": 0.59855957, "grad_norm": 0.62999094, "learning_rate": 1.46e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01576, "epoch": 1.5988791, "global_step/max_steps": "500/936", "percentage": "53.42%", "elapsed_time": "8h 44m 21s", "remaining_time": "7h 37m 14s"}
+{"loss": 0.55943136, "grad_norm": 0.62793673, "learning_rate": 1.434e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015763, "epoch": 1.61489191, "global_step/max_steps": "505/936", "percentage": "53.95%", "elapsed_time": "8h 49m 31s", "remaining_time": "7h 31m 56s"}
+{"loss": 0.55001945, "grad_norm": 0.64144078, "learning_rate": 1.407e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015768, "epoch": 1.63090472, "global_step/max_steps": "510/936", "percentage": "54.49%", "elapsed_time": "8h 54m 39s", "remaining_time": "7h 26m 36s"}
+{"loss": 0.57425137, "grad_norm": 0.65977768, "learning_rate": 1.381e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.01577, "epoch": 1.64691753, "global_step/max_steps": "515/936", "percentage": "55.02%", "elapsed_time": "8h 59m 52s", "remaining_time": "7h 21m 19s"}
+{"loss": 0.56872339, "grad_norm": 0.59814443, "learning_rate": 1.354e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015773, "epoch": 1.66293034, "global_step/max_steps": "520/936", "percentage": "55.56%", "elapsed_time": "9h 5m 2s", "remaining_time": "7h 16m 1s"}
+{"loss": 0.5612329, "grad_norm": 0.6626789, "learning_rate": 1.328e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015776, "epoch": 1.67894315, "global_step/max_steps": "525/936", "percentage": "56.09%", "elapsed_time": "9h 10m 13s", "remaining_time": "7h 10m 44s"}
+{"loss": 0.55641694, "grad_norm": 0.6165757, "learning_rate": 1.302e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015778, "epoch": 1.69495596, "global_step/max_steps": "530/936", "percentage": "56.62%", "elapsed_time": "9h 15m 26s", "remaining_time": "7h 5m 29s"}
+{"loss": 0.59409752, "grad_norm": 0.65576318, "learning_rate": 1.276e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015782, "epoch": 1.71096878, "global_step/max_steps": "535/936", "percentage": "57.16%", "elapsed_time": "9h 20m 33s", "remaining_time": "7h 0m 9s"}
+{"loss": 0.55875688, "grad_norm": 0.62465309, "learning_rate": 1.249e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015784, "epoch": 1.72698159, "global_step/max_steps": "540/936", "percentage": "57.69%", "elapsed_time": "9h 25m 47s", "remaining_time": "6h 54m 55s"}
+{"loss": 0.52943115, "grad_norm": 0.56887063, "learning_rate": 1.223e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015785, "epoch": 1.7429944, "global_step/max_steps": "545/936", "percentage": "58.23%", "elapsed_time": "9h 31m 2s", "remaining_time": "6h 49m 41s"}
+{"loss": 0.55800018, "grad_norm": 0.66950407, "learning_rate": 1.197e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015787, "epoch": 1.75900721, "global_step/max_steps": "550/936", "percentage": "58.76%", "elapsed_time": "9h 36m 13s", "remaining_time": "6h 44m 24s"}
+{"loss": 0.57581358, "grad_norm": 0.62353001, "learning_rate": 1.171e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015788, "epoch": 1.77502002, "global_step/max_steps": "555/936", "percentage": "59.29%", "elapsed_time": "9h 41m 29s", "remaining_time": "6h 39m 11s"}
+{"loss": 0.57144208, "grad_norm": 0.63526118, "learning_rate": 1.146e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015791, "epoch": 1.79103283, "global_step/max_steps": "560/936", "percentage": "59.83%", "elapsed_time": "9h 46m 39s", "remaining_time": "6h 33m 53s"}
+{"loss": 0.55343156, "grad_norm": 0.68069869, "learning_rate": 1.12e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015793, "epoch": 1.80704564, "global_step/max_steps": "565/936", "percentage": "60.36%", "elapsed_time": "9h 51m 49s", "remaining_time": "6h 28m 36s"}
+{"loss": 0.54114871, "grad_norm": 0.59935772, "learning_rate": 1.094e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015794, "epoch": 1.82305845, "global_step/max_steps": "570/936", "percentage": "60.90%", "elapsed_time": "9h 57m 4s", "remaining_time": "6h 23m 23s"}
+{"loss": 0.54671941, "grad_norm": 0.66805905, "learning_rate": 1.069e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015797, "epoch": 1.83907126, "global_step/max_steps": "575/936", "percentage": "61.43%", "elapsed_time": "10h 2m 15s", "remaining_time": "6h 18m 6s"}
+{"loss": 0.57824945, "grad_norm": 0.66497161, "learning_rate": 1.049e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.0158, "epoch": 1.85508407, "global_step/max_steps": "580/936", "percentage": "61.97%", "elapsed_time": "10h 7m 24s", "remaining_time": "6h 12m 49s"}
+{"loss": 0.57358356, "grad_norm": 0.61038546, "learning_rate": 1.023e-05, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015801, "epoch": 1.87109688, "global_step/max_steps": "585/936", "percentage": "62.50%", "elapsed_time": "10h 12m 38s", "remaining_time": "6h 7m 34s"}
+{"loss": 0.54877539, "grad_norm": 0.75307211, "learning_rate": 9.98e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015805, "epoch": 1.88710969, "global_step/max_steps": "590/936", "percentage": "63.03%", "elapsed_time": "10h 17m 45s", "remaining_time": "6h 2m 16s"}
+{"loss": 0.57379742, "grad_norm": 0.61708442, "learning_rate": 9.73e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015808, "epoch": 1.9031225, "global_step/max_steps": "595/936", "percentage": "63.57%", "elapsed_time": "10h 22m 55s", "remaining_time": "5h 57m 0s"}
+{"loss": 0.55204997, "grad_norm": 0.64517944, "learning_rate": 9.49e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015809, "epoch": 1.91913531, "global_step/max_steps": "600/936", "percentage": "64.10%", "elapsed_time": "10h 28m 8s", "remaining_time": "5h 51m 45s"}
+{"loss": 0.54695177, "grad_norm": 0.62519423, "learning_rate": 9.24e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015812, "epoch": 1.93514812, "global_step/max_steps": "605/936", "percentage": "64.64%", "elapsed_time": "10h 33m 18s", "remaining_time": "5h 46m 29s"}
+{"loss": 0.56501989, "grad_norm": 0.61982048, "learning_rate": 9e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015812, "epoch": 1.95116093, "global_step/max_steps": "610/936", "percentage": "65.17%", "elapsed_time": "10h 38m 33s", "remaining_time": "5h 41m 15s"}
+{"loss": 0.53078766, "grad_norm": 0.63561612, "learning_rate": 8.76e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015813, "epoch": 1.96717374, "global_step/max_steps": "615/936", "percentage": "65.71%", "elapsed_time": "10h 43m 47s", "remaining_time": "5h 36m 1s"}
+{"loss": 0.55520983, "grad_norm": 0.66011745, "learning_rate": 8.52e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015815, "epoch": 1.98318655, "global_step/max_steps": "620/936", "percentage": "66.24%", "elapsed_time": "10h 48m 58s", "remaining_time": "5h 30m 46s"}
+{"loss": 0.55757914, "grad_norm": 0.64936355, "learning_rate": 8.28e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015818, "epoch": 1.99919936, "global_step/max_steps": "625/936", "percentage": "66.77%", "elapsed_time": "10h 54m 5s", "remaining_time": "5h 25m 28s"}
+{"loss": 0.52105627, "grad_norm": 0.61302059, "learning_rate": 8.04e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015839, "epoch": 2.01281025, "global_step/max_steps": "630/936", "percentage": "67.31%", "elapsed_time": "10h 58m 31s", "remaining_time": "5h 19m 51s"}
+{"loss": 0.53484406, "grad_norm": 0.61695648, "learning_rate": 7.81e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015839, "epoch": 2.02882306, "global_step/max_steps": "635/936", "percentage": "67.84%", "elapsed_time": "11h 3m 46s", "remaining_time": "5h 14m 38s"}
+{"loss": 0.48503141, "grad_norm": 0.65024846, "learning_rate": 7.58e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015839, "epoch": 2.04483587, "global_step/max_steps": "640/936", "percentage": "68.38%", "elapsed_time": "11h 9m 1s", "remaining_time": "5h 9m 25s"}
+{"loss": 0.52498808, "grad_norm": 0.6899964, "learning_rate": 7.35e-06, "memory(GiB)": 21.89, "train_speed(iter/s)": 0.015787, "epoch": 2.06084868, "global_step/max_steps": "645/936", "percentage": "68.91%", "elapsed_time": "11h 16m 31s", "remaining_time": "5h 5m 13s"}

v0-20250526-142723/runs/events.out.tfevents.1748269983.e0b00eb95078.491.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06fc0271b4f4505a579610cf7d1d0655171607267cc56b67c8cbdcfbed850035
+size 50114