Upload folder using huggingface_hub

.gitattributes

@@ -40,3 +40,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 7b_cold_start_fixUnderfitting/checkpoint-epoch5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 7b_cold_start_fixUnderfitting/checkpoint-epoch6/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 7b_cold_start_fixUnderfitting/checkpoint-epoch7/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+7b_cold_start_fixUnderfitting/checkpoint-epoch8/tokenizer.json filter=lfs diff=lfs merge=lfs -text

7b_cold_start_fixUnderfitting/checkpoint-epoch8/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/config.json

@@ -0,0 +1,65 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 28,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 3584,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 152064
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "attn_implementation": "flash_attention_2",
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.51.3"
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/latest

@@ -0,0 +1 @@
+global_step320

7b_cold_start_fixUnderfitting/checkpoint-epoch8/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4006b8a539b116b029e1039004352df5d34ddaa46fb509a1049740f755148961
+size 4968243304

7b_cold_start_fixUnderfitting/checkpoint-epoch8/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1830db6908dcc76df3a71492acbcf2b8cac130114cf1f3c2d9edae8de8c6de3
+size 4991495816

7b_cold_start_fixUnderfitting/checkpoint-epoch8/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09c1807c6d00d7cab94f7db39d4c02ebb8537225ccde383861ac48db97945aa6
+size 4932751040

7b_cold_start_fixUnderfitting/checkpoint-epoch8/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dd068336d14d45ffb43cef374d286cc6ba9d8741b028f90a7d040d847961f4a
+size 1691924384

7b_cold_start_fixUnderfitting/checkpoint-epoch8/model.safetensors.index.json

@@ -0,0 +1,736 @@
+{
+  "metadata": {
+    "total_size": 16584333312
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors",
+    "visual.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.gate_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
+    "visual.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.ln_q.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.0.bias": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.0.weight": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.2.bias": "model-00001-of-00004.safetensors",
+    "visual.merger.mlp.2.weight": "model-00001-of-00004.safetensors",
+    "visual.patch_embed.proj.weight": "model-00001-of-00004.safetensors"
+  }
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ee195ebde9bf012f945f068f133e7fe22fef5450c496607e3ef11cc2034a186
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf0fe1a3315d60b197207c5cb249d0ce4f9ce6d7585e696276d9ffbcb5379893
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01c5bd6eae04542162b3e94245555bd81312524066bc01d0ebbfc4fd8554240e
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45b74942c68b00d657cfce186b0eeb4aa8f52efa04b114803b605fee8de45972
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cd66dd2ba958fc9929441817d8154abbd929c0aa9cd66ff3171965bdaaf5d78
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89eeedefdd62514d0130acc330a5c08e9774c95d38c60997905cfd65fc54b710
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f43ced939100082608f57561a10e1888e69210c80675068db530c5815889910e
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d8d6ee244d99525e7004ae3f02d44ae63082d81fbbab7306f641ac6aeeb736f
+size 15984

7b_cold_start_fixUnderfitting/checkpoint-epoch8/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d90a86bcba75babdb72096b198b11b8ebc0d5bcaf610d632b84ed46b01a3e3b
+size 1064

7b_cold_start_fixUnderfitting/checkpoint-epoch8/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

7b_cold_start_fixUnderfitting/checkpoint-epoch8/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/trainer_state.json

@@ -0,0 +1,2274 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 8.0,
+  "eval_steps": 500,
+  "global_step": 320,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.025,
+      "grad_norm": 2.1026898827355764,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 1.6621,
+      "step": 1
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.6413522165034502,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 1.7318,
+      "step": 2
+    },
+    {
+      "epoch": 0.075,
+      "grad_norm": 0.6085035314989437,
+      "learning_rate": 2.5e-06,
+      "loss": 1.6973,
+      "step": 3
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.6097206000487838,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 1.7155,
+      "step": 4
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 0.6181111292344054,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 1.7199,
+      "step": 5
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.6095519945061052,
+      "learning_rate": 5e-06,
+      "loss": 1.7225,
+      "step": 6
+    },
+    {
+      "epoch": 0.175,
+      "grad_norm": 0.5993146111892984,
+      "learning_rate": 5.833333333333334e-06,
+      "loss": 1.6788,
+      "step": 7
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.6168594833390011,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 1.6941,
+      "step": 8
+    },
+    {
+      "epoch": 0.225,
+      "grad_norm": 0.649510684690767,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 1.7237,
+      "step": 9
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.6127108674374603,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 1.6575,
+      "step": 10
+    },
+    {
+      "epoch": 0.275,
+      "grad_norm": 0.6444847916730025,
+      "learning_rate": 9.166666666666666e-06,
+      "loss": 1.6477,
+      "step": 11
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.6274944995465307,
+      "learning_rate": 1e-05,
+      "loss": 1.6443,
+      "step": 12
+    },
+    {
+      "epoch": 0.325,
+      "grad_norm": 0.6543139619848919,
+      "learning_rate": 9.999836101895782e-06,
+      "loss": 1.6202,
+      "step": 13
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.8235053540937074,
+      "learning_rate": 9.999344418328161e-06,
+      "loss": 1.6082,
+      "step": 14
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 0.9775219557515483,
+      "learning_rate": 9.99852498153154e-06,
+      "loss": 1.5583,
+      "step": 15
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.1030636654770976,
+      "learning_rate": 9.997377845227577e-06,
+      "loss": 1.5679,
+      "step": 16
+    },
+    {
+      "epoch": 0.425,
+      "grad_norm": 1.533566842443126,
+      "learning_rate": 9.995903084621651e-06,
+      "loss": 1.5359,
+      "step": 17
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 1.718620417025083,
+      "learning_rate": 9.994100796397954e-06,
+      "loss": 1.4736,
+      "step": 18
+    },
+    {
+      "epoch": 0.475,
+      "grad_norm": 1.848832331677319,
+      "learning_rate": 9.991971098713136e-06,
+      "loss": 1.4416,
+      "step": 19
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 1.1378030725614972,
+      "learning_rate": 9.98951413118856e-06,
+      "loss": 1.4385,
+      "step": 20
+    },
+    {
+      "epoch": 0.525,
+      "grad_norm": 0.7491289081750082,
+      "learning_rate": 9.986730054901153e-06,
+      "loss": 1.3793,
+      "step": 21
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.6818788464479123,
+      "learning_rate": 9.983619052372847e-06,
+      "loss": 1.368,
+      "step": 22
+    },
+    {
+      "epoch": 0.575,
+      "grad_norm": 0.6341413849629475,
+      "learning_rate": 9.98018132755861e-06,
+      "loss": 1.3467,
+      "step": 23
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.5923369379348106,
+      "learning_rate": 9.97641710583307e-06,
+      "loss": 1.3555,
+      "step": 24
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 0.5340471860637729,
+      "learning_rate": 9.972326633975754e-06,
+      "loss": 1.349,
+      "step": 25
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.4785970184408438,
+      "learning_rate": 9.96791018015489e-06,
+      "loss": 1.3337,
+      "step": 26
+    },
+    {
+      "epoch": 0.675,
+      "grad_norm": 0.4414536685738152,
+      "learning_rate": 9.963168033909843e-06,
+      "loss": 1.3334,
+      "step": 27
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.4860929708817065,
+      "learning_rate": 9.958100506132127e-06,
+      "loss": 1.3159,
+      "step": 28
+    },
+    {
+      "epoch": 0.725,
+      "grad_norm": 0.45181979183711657,
+      "learning_rate": 9.952707929045018e-06,
+      "loss": 1.3231,
+      "step": 29
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.4400284453917975,
+      "learning_rate": 9.946990656181782e-06,
+      "loss": 1.2993,
+      "step": 30
+    },
+    {
+      "epoch": 0.775,
+      "grad_norm": 0.38337739876412114,
+      "learning_rate": 9.94094906236249e-06,
+      "loss": 1.2952,
+      "step": 31
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.37181863514378,
+      "learning_rate": 9.934583543669454e-06,
+      "loss": 1.3109,
+      "step": 32
+    },
+    {
+      "epoch": 0.825,
+      "grad_norm": 0.3401881416689356,
+      "learning_rate": 9.927894517421252e-06,
+      "loss": 1.3157,
+      "step": 33
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.3239891568707606,
+      "learning_rate": 9.920882422145372e-06,
+      "loss": 1.34,
+      "step": 34
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.3033475087643797,
+      "learning_rate": 9.913547717549463e-06,
+      "loss": 1.283,
+      "step": 35
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.2888152197430936,
+      "learning_rate": 9.905890884491196e-06,
+      "loss": 1.2892,
+      "step": 36
+    },
+    {
+      "epoch": 0.925,
+      "grad_norm": 0.28458390089996766,
+      "learning_rate": 9.897912424946741e-06,
+      "loss": 1.2646,
+      "step": 37
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.27339498391168426,
+      "learning_rate": 9.889612861977855e-06,
+      "loss": 1.2832,
+      "step": 38
+    },
+    {
+      "epoch": 0.975,
+      "grad_norm": 0.27940221076287475,
+      "learning_rate": 9.88099273969759e-06,
+      "loss": 1.2422,
+      "step": 39
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.27720461090670556,
+      "learning_rate": 9.872052623234632e-06,
+      "loss": 1.2919,
+      "step": 40
+    },
+    {
+      "epoch": 1.025,
+      "grad_norm": 0.25894847092459033,
+      "learning_rate": 9.86279309869623e-06,
+      "loss": 1.2494,
+      "step": 41
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 0.2824528364474668,
+      "learning_rate": 9.853214773129796e-06,
+      "loss": 1.2404,
+      "step": 42
+    },
+    {
+      "epoch": 1.075,
+      "grad_norm": 0.2731217503169999,
+      "learning_rate": 9.843318274483089e-06,
+      "loss": 1.2762,
+      "step": 43
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.282165604906485,
+      "learning_rate": 9.833104251563058e-06,
+      "loss": 1.2373,
+      "step": 44
+    },
+    {
+      "epoch": 1.125,
+      "grad_norm": 0.27542846366926443,
+      "learning_rate": 9.822573373993296e-06,
+      "loss": 1.2467,
+      "step": 45
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 0.2851918766199106,
+      "learning_rate": 9.811726332170153e-06,
+      "loss": 1.267,
+      "step": 46
+    },
+    {
+      "epoch": 1.175,
+      "grad_norm": 0.2599062629698892,
+      "learning_rate": 9.800563837217465e-06,
+      "loss": 1.2585,
+      "step": 47
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.2586420496872339,
+      "learning_rate": 9.789086620939936e-06,
+      "loss": 1.2173,
+      "step": 48
+    },
+    {
+      "epoch": 1.225,
+      "grad_norm": 0.27264444944219546,
+      "learning_rate": 9.777295435775165e-06,
+      "loss": 1.1856,
+      "step": 49
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.2648452730214859,
+      "learning_rate": 9.765191054744305e-06,
+      "loss": 1.2354,
+      "step": 50
+    },
+    {
+      "epoch": 1.275,
+      "grad_norm": 0.27625402001557403,
+      "learning_rate": 9.752774271401403e-06,
+      "loss": 1.1857,
+      "step": 51
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.29532784079485663,
+      "learning_rate": 9.740045899781353e-06,
+      "loss": 1.2058,
+      "step": 52
+    },
+    {
+      "epoch": 1.325,
+      "grad_norm": 0.2701426980802868,
+      "learning_rate": 9.727006774346551e-06,
+      "loss": 1.1724,
+      "step": 53
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 0.26735542553762237,
+      "learning_rate": 9.713657749932172e-06,
+      "loss": 1.194,
+      "step": 54
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 0.24830074105443878,
+      "learning_rate": 9.699999701690133e-06,
+      "loss": 1.1969,
+      "step": 55
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.2511171777018325,
+      "learning_rate": 9.68603352503172e-06,
+      "loss": 1.1822,
+      "step": 56
+    },
+    {
+      "epoch": 1.425,
+      "grad_norm": 0.25711859387852454,
+      "learning_rate": 9.671760135568882e-06,
+      "loss": 1.1941,
+      "step": 57
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 0.24983954826654237,
+      "learning_rate": 9.657180469054213e-06,
+      "loss": 1.1597,
+      "step": 58
+    },
+    {
+      "epoch": 1.475,
+      "grad_norm": 0.2507740474880207,
+      "learning_rate": 9.64229548131959e-06,
+      "loss": 1.1631,
+      "step": 59
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.23964630893140215,
+      "learning_rate": 9.627106148213521e-06,
+      "loss": 1.173,
+      "step": 60
+    },
+    {
+      "epoch": 1.525,
+      "grad_norm": 0.22971666963865991,
+      "learning_rate": 9.61161346553717e-06,
+      "loss": 1.1446,
+      "step": 61
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 0.22847059454081425,
+      "learning_rate": 9.595818448979061e-06,
+      "loss": 1.1692,
+      "step": 62
+    },
+    {
+      "epoch": 1.575,
+      "grad_norm": 0.2544697716980138,
+      "learning_rate": 9.579722134048506e-06,
+      "loss": 1.1712,
+      "step": 63
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.2248726099006599,
+      "learning_rate": 9.563325576007702e-06,
+      "loss": 1.1523,
+      "step": 64
+    },
+    {
+      "epoch": 1.625,
+      "grad_norm": 0.23257460453519807,
+      "learning_rate": 9.546629849802562e-06,
+      "loss": 1.146,
+      "step": 65
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 0.2400345523124048,
+      "learning_rate": 9.529636049992235e-06,
+      "loss": 1.1238,
+      "step": 66
+    },
+    {
+      "epoch": 1.675,
+      "grad_norm": 0.2410972327939822,
+      "learning_rate": 9.51234529067735e-06,
+      "loss": 1.1745,
+      "step": 67
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.2395032264738681,
+      "learning_rate": 9.494758705426978e-06,
+      "loss": 1.156,
+      "step": 68
+    },
+    {
+      "epoch": 1.725,
+      "grad_norm": 0.23423115235391723,
+      "learning_rate": 9.476877447204308e-06,
+      "loss": 1.1413,
+      "step": 69
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 0.24379065472217767,
+      "learning_rate": 9.458702688291072e-06,
+      "loss": 1.1677,
+      "step": 70
+    },
+    {
+      "epoch": 1.775,
+      "grad_norm": 0.24016354424050318,
+      "learning_rate": 9.440235620210682e-06,
+      "loss": 1.1235,
+      "step": 71
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.25909751239656786,
+      "learning_rate": 9.421477453650118e-06,
+      "loss": 1.1139,
+      "step": 72
+    },
+    {
+      "epoch": 1.825,
+      "grad_norm": 0.24033129150490448,
+      "learning_rate": 9.402429418380555e-06,
+      "loss": 1.1582,
+      "step": 73
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 0.2413776856942996,
+      "learning_rate": 9.38309276317674e-06,
+      "loss": 1.1259,
+      "step": 74
+    },
+    {
+      "epoch": 1.875,
+      "grad_norm": 0.24663210297989538,
+      "learning_rate": 9.363468755735123e-06,
+      "loss": 1.0843,
+      "step": 75
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.2469034876268094,
+      "learning_rate": 9.343558682590757e-06,
+      "loss": 1.1077,
+      "step": 76
+    },
+    {
+      "epoch": 1.925,
+      "grad_norm": 0.25810333557343196,
+      "learning_rate": 9.323363849032933e-06,
+      "loss": 1.1145,
+      "step": 77
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 0.2457214465581943,
+      "learning_rate": 9.302885579019626e-06,
+      "loss": 1.119,
+      "step": 78
+    },
+    {
+      "epoch": 1.975,
+      "grad_norm": 0.2530172029554143,
+      "learning_rate": 9.282125215090694e-06,
+      "loss": 1.1324,
+      "step": 79
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.26171895830403796,
+      "learning_rate": 9.261084118279846e-06,
+      "loss": 1.0914,
+      "step": 80
+    },
+    {
+      "epoch": 2.025,
+      "grad_norm": 0.2596448526704054,
+      "learning_rate": 9.23976366802544e-06,
+      "loss": 1.1202,
+      "step": 81
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 0.2575246492390129,
+      "learning_rate": 9.218165262080024e-06,
+      "loss": 1.1191,
+      "step": 82
+    },
+    {
+      "epoch": 2.075,
+      "grad_norm": 0.24809186590262497,
+      "learning_rate": 9.196290316418712e-06,
+      "loss": 1.0852,
+      "step": 83
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 0.25529825764524233,
+      "learning_rate": 9.174140265146356e-06,
+      "loss": 1.1091,
+      "step": 84
+    },
+    {
+      "epoch": 2.125,
+      "grad_norm": 0.2598333152137317,
+      "learning_rate": 9.151716560403521e-06,
+      "loss": 1.0512,
+      "step": 85
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 0.2533838531424551,
+      "learning_rate": 9.129020672271283e-06,
+      "loss": 1.0862,
+      "step": 86
+    },
+    {
+      "epoch": 2.175,
+      "grad_norm": 0.2504337844095705,
+      "learning_rate": 9.106054088674861e-06,
+      "loss": 1.0717,
+      "step": 87
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.25443455111995733,
+      "learning_rate": 9.082818315286054e-06,
+      "loss": 1.0658,
+      "step": 88
+    },
+    {
+      "epoch": 2.225,
+      "grad_norm": 0.26694514174541256,
+      "learning_rate": 9.059314875424553e-06,
+      "loss": 1.0228,
+      "step": 89
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 0.26148960189209175,
+      "learning_rate": 9.035545309958048e-06,
+      "loss": 1.052,
+      "step": 90
+    },
+    {
+      "epoch": 2.275,
+      "grad_norm": 0.27532917589486194,
+      "learning_rate": 9.011511177201226e-06,
+      "loss": 1.07,
+      "step": 91
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 0.28730068505165196,
+      "learning_rate": 8.987214052813605e-06,
+      "loss": 1.0609,
+      "step": 92
+    },
+    {
+      "epoch": 2.325,
+      "grad_norm": 0.2698216580724319,
+      "learning_rate": 8.962655529696236e-06,
+      "loss": 1.0579,
+      "step": 93
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 0.24786365341013725,
+      "learning_rate": 8.937837217887273e-06,
+      "loss": 1.0452,
+      "step": 94
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.24197792160968903,
+      "learning_rate": 8.912760744456416e-06,
+      "loss": 1.0346,
+      "step": 95
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.25871618240180544,
+      "learning_rate": 8.887427753398249e-06,
+      "loss": 1.0452,
+      "step": 96
+    },
+    {
+      "epoch": 2.425,
+      "grad_norm": 0.2670995555409698,
+      "learning_rate": 8.861839905524452e-06,
+      "loss": 1.004,
+      "step": 97
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 0.26240119221560043,
+      "learning_rate": 8.83599887835493e-06,
+      "loss": 1.0381,
+      "step": 98
+    },
+    {
+      "epoch": 2.475,
+      "grad_norm": 0.25007005933407084,
+      "learning_rate": 8.809906366007832e-06,
+      "loss": 1.0237,
+      "step": 99
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.27210733980893814,
+      "learning_rate": 8.783564079088478e-06,
+      "loss": 1.0326,
+      "step": 100
+    },
+    {
+      "epoch": 2.525,
+      "grad_norm": 0.2778485282051736,
+      "learning_rate": 8.756973744577221e-06,
+      "loss": 1.0213,
+      "step": 101
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 0.2709595058080484,
+      "learning_rate": 8.730137105716231e-06,
+      "loss": 1.0046,
+      "step": 102
+    },
+    {
+      "epoch": 2.575,
+      "grad_norm": 0.2723123534955665,
+      "learning_rate": 8.7030559218952e-06,
+      "loss": 1.0572,
+      "step": 103
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 0.26318374858426913,
+      "learning_rate": 8.675731968536004e-06,
+      "loss": 1.0142,
+      "step": 104
+    },
+    {
+      "epoch": 2.625,
+      "grad_norm": 0.26423478733108197,
+      "learning_rate": 8.648167036976303e-06,
+      "loss": 1.0083,
+      "step": 105
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 0.2527490323367728,
+      "learning_rate": 8.620362934352109e-06,
+      "loss": 1.0152,
+      "step": 106
+    },
+    {
+      "epoch": 2.675,
+      "grad_norm": 0.2817733909459089,
+      "learning_rate": 8.592321483479305e-06,
+      "loss": 0.9994,
+      "step": 107
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 0.25088178573538616,
+      "learning_rate": 8.564044522734147e-06,
+      "loss": 0.966,
+      "step": 108
+    },
+    {
+      "epoch": 2.725,
+      "grad_norm": 0.25165839780166765,
+      "learning_rate": 8.535533905932739e-06,
+      "loss": 0.9897,
+      "step": 109
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.27123004373609994,
+      "learning_rate": 8.506791502209497e-06,
+      "loss": 1.0011,
+      "step": 110
+    },
+    {
+      "epoch": 2.775,
+      "grad_norm": 0.25885979274531185,
+      "learning_rate": 8.477819195894615e-06,
+      "loss": 1.0375,
+      "step": 111
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.25413119834372133,
+      "learning_rate": 8.448618886390523e-06,
+      "loss": 0.974,
+      "step": 112
+    },
+    {
+      "epoch": 2.825,
+      "grad_norm": 0.26932379526063627,
+      "learning_rate": 8.419192488047369e-06,
+      "loss": 0.9948,
+      "step": 113
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 0.26954131129733855,
+      "learning_rate": 8.389541930037516e-06,
+      "loss": 1.0171,
+      "step": 114
+    },
+    {
+      "epoch": 2.875,
+      "grad_norm": 0.2667631834386252,
+      "learning_rate": 8.359669156229062e-06,
+      "loss": 0.9917,
+      "step": 115
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 0.2875114899558342,
+      "learning_rate": 8.329576125058406e-06,
+      "loss": 0.9758,
+      "step": 116
+    },
+    {
+      "epoch": 2.925,
+      "grad_norm": 0.28675332351961746,
+      "learning_rate": 8.29926480940185e-06,
+      "loss": 0.963,
+      "step": 117
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 0.3013638054236219,
+      "learning_rate": 8.268737196446264e-06,
+      "loss": 0.9859,
+      "step": 118
+    },
+    {
+      "epoch": 2.975,
+      "grad_norm": 0.30601615118396874,
+      "learning_rate": 8.237995287558801e-06,
+      "loss": 0.9685,
+      "step": 119
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.30346948632781146,
+      "learning_rate": 8.207041098155701e-06,
+      "loss": 0.9872,
+      "step": 120
+    },
+    {
+      "epoch": 3.025,
+      "grad_norm": 0.31531631458437354,
+      "learning_rate": 8.175876657570142e-06,
+      "loss": 0.964,
+      "step": 121
+    },
+    {
+      "epoch": 3.05,
+      "grad_norm": 0.29114779511016237,
+      "learning_rate": 8.144504008919224e-06,
+      "loss": 0.9655,
+      "step": 122
+    },
+    {
+      "epoch": 3.075,
+      "grad_norm": 0.32729676853704537,
+      "learning_rate": 8.112925208969996e-06,
+      "loss": 0.9484,
+      "step": 123
+    },
+    {
+      "epoch": 3.1,
+      "grad_norm": 0.2968428193830138,
+      "learning_rate": 8.081142328004638e-06,
+      "loss": 0.9885,
+      "step": 124
+    },
+    {
+      "epoch": 3.125,
+      "grad_norm": 0.31850537244671423,
+      "learning_rate": 8.049157449684723e-06,
+      "loss": 0.9593,
+      "step": 125
+    },
+    {
+      "epoch": 3.15,
+      "grad_norm": 0.30145179934590266,
+      "learning_rate": 8.016972670914624e-06,
+      "loss": 0.9443,
+      "step": 126
+    },
+    {
+      "epoch": 3.175,
+      "grad_norm": 0.32941902019125735,
+      "learning_rate": 7.984590101704026e-06,
+      "loss": 0.9411,
+      "step": 127
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 0.33230400688084355,
+      "learning_rate": 7.952011865029614e-06,
+      "loss": 0.9353,
+      "step": 128
+    },
+    {
+      "epoch": 3.225,
+      "grad_norm": 0.31073576739762815,
+      "learning_rate": 7.919240096695877e-06,
+      "loss": 0.9403,
+      "step": 129
+    },
+    {
+      "epoch": 3.25,
+      "grad_norm": 0.316466603610876,
+      "learning_rate": 7.886276945195098e-06,
+      "loss": 0.9575,
+      "step": 130
+    },
+    {
+      "epoch": 3.275,
+      "grad_norm": 0.3252873314387951,
+      "learning_rate": 7.853124571566491e-06,
+      "loss": 0.9476,
+      "step": 131
+    },
+    {
+      "epoch": 3.3,
+      "grad_norm": 0.34237602645794485,
+      "learning_rate": 7.819785149254534e-06,
+      "loss": 0.8938,
+      "step": 132
+    },
+    {
+      "epoch": 3.325,
+      "grad_norm": 0.3386834141148311,
+      "learning_rate": 7.786260863966468e-06,
+      "loss": 0.8937,
+      "step": 133
+    },
+    {
+      "epoch": 3.35,
+      "grad_norm": 0.32892426523618096,
+      "learning_rate": 7.752553913529019e-06,
+      "loss": 0.9226,
+      "step": 134
+    },
+    {
+      "epoch": 3.375,
+      "grad_norm": 0.3663645071149253,
+      "learning_rate": 7.718666507744293e-06,
+      "loss": 0.9331,
+      "step": 135
+    },
+    {
+      "epoch": 3.4,
+      "grad_norm": 0.35492405802228927,
+      "learning_rate": 7.68460086824492e-06,
+      "loss": 0.9024,
+      "step": 136
+    },
+    {
+      "epoch": 3.425,
+      "grad_norm": 0.3461059129442488,
+      "learning_rate": 7.65035922834839e-06,
+      "loss": 0.9118,
+      "step": 137
+    },
+    {
+      "epoch": 3.45,
+      "grad_norm": 0.3674770015752456,
+      "learning_rate": 7.61594383291065e-06,
+      "loss": 0.885,
+      "step": 138
+    },
+    {
+      "epoch": 3.475,
+      "grad_norm": 0.29868653659173194,
+      "learning_rate": 7.58135693817893e-06,
+      "loss": 0.8795,
+      "step": 139
+    },
+    {
+      "epoch": 3.5,
+      "grad_norm": 0.396103206356086,
+      "learning_rate": 7.546600811643816e-06,
+      "loss": 0.9057,
+      "step": 140
+    },
+    {
+      "epoch": 3.525,
+      "grad_norm": 0.4091265454754814,
+      "learning_rate": 7.5116777318906105e-06,
+      "loss": 0.9045,
+      "step": 141
+    },
+    {
+      "epoch": 3.55,
+      "grad_norm": 0.34738555176640296,
+      "learning_rate": 7.476589988449939e-06,
+      "loss": 0.897,
+      "step": 142
+    },
+    {
+      "epoch": 3.575,
+      "grad_norm": 0.3381371740638481,
+      "learning_rate": 7.441339881647654e-06,
+      "loss": 0.9232,
+      "step": 143
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 0.35909644765143234,
+      "learning_rate": 7.405929722454026e-06,
+      "loss": 0.8787,
+      "step": 144
+    },
+    {
+      "epoch": 3.625,
+      "grad_norm": 0.3825665161255983,
+      "learning_rate": 7.370361832332241e-06,
+      "loss": 0.9113,
+      "step": 145
+    },
+    {
+      "epoch": 3.65,
+      "grad_norm": 0.30851922400853754,
+      "learning_rate": 7.334638543086203e-06,
+      "loss": 0.88,
+      "step": 146
+    },
+    {
+      "epoch": 3.675,
+      "grad_norm": 0.37247312676947136,
+      "learning_rate": 7.298762196707668e-06,
+      "loss": 0.874,
+      "step": 147
+    },
+    {
+      "epoch": 3.7,
+      "grad_norm": 0.38511911071863114,
+      "learning_rate": 7.262735145222696e-06,
+      "loss": 0.8783,
+      "step": 148
+    },
+    {
+      "epoch": 3.725,
+      "grad_norm": 0.45755114413180425,
+      "learning_rate": 7.226559750537462e-06,
+      "loss": 0.884,
+      "step": 149
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 0.30880788937024484,
+      "learning_rate": 7.190238384283413e-06,
+      "loss": 0.9085,
+      "step": 150
+    },
+    {
+      "epoch": 3.775,
+      "grad_norm": 0.33118326548212185,
+      "learning_rate": 7.1537734276617744e-06,
+      "loss": 0.8671,
+      "step": 151
+    },
+    {
+      "epoch": 3.8,
+      "grad_norm": 0.41391364228528094,
+      "learning_rate": 7.117167271287453e-06,
+      "loss": 0.8695,
+      "step": 152
+    },
+    {
+      "epoch": 3.825,
+      "grad_norm": 0.41995379444939823,
+      "learning_rate": 7.080422315032298e-06,
+      "loss": 0.8989,
+      "step": 153
+    },
+    {
+      "epoch": 3.85,
+      "grad_norm": 0.33781129667429866,
+      "learning_rate": 7.043540967867782e-06,
+      "loss": 0.8927,
+      "step": 154
+    },
+    {
+      "epoch": 3.875,
+      "grad_norm": 0.37184734206990244,
+      "learning_rate": 7.006525647707054e-06,
+      "loss": 0.8986,
+      "step": 155
+    },
+    {
+      "epoch": 3.9,
+      "grad_norm": 0.42831001354188236,
+      "learning_rate": 6.969378781246436e-06,
+      "loss": 0.8899,
+      "step": 156
+    },
+    {
+      "epoch": 3.925,
+      "grad_norm": 0.37314784171511645,
+      "learning_rate": 6.932102803806324e-06,
+      "loss": 0.857,
+      "step": 157
+    },
+    {
+      "epoch": 3.95,
+      "grad_norm": 0.38265810168742836,
+      "learning_rate": 6.894700159171535e-06,
+      "loss": 0.8423,
+      "step": 158
+    },
+    {
+      "epoch": 3.975,
+      "grad_norm": 0.4345836191641771,
+      "learning_rate": 6.857173299431084e-06,
+      "loss": 0.8487,
+      "step": 159
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.28712745976233395,
+      "learning_rate": 6.819524684817439e-06,
+      "loss": 0.8494,
+      "step": 160
+    },
+    {
+      "epoch": 4.025,
+      "grad_norm": 0.363373067735039,
+      "learning_rate": 6.781756783545225e-06,
+      "loss": 0.8516,
+      "step": 161
+    },
+    {
+      "epoch": 4.05,
+      "grad_norm": 0.38168917727729257,
+      "learning_rate": 6.743872071649411e-06,
+      "loss": 0.8882,
+      "step": 162
+    },
+    {
+      "epoch": 4.075,
+      "grad_norm": 0.39729085134853515,
+      "learning_rate": 6.705873032822981e-06,
+      "loss": 0.8337,
+      "step": 163
+    },
+    {
+      "epoch": 4.1,
+      "grad_norm": 0.37813465835851745,
+      "learning_rate": 6.667762158254104e-06,
+      "loss": 0.8309,
+      "step": 164
+    },
+    {
+      "epoch": 4.125,
+      "grad_norm": 0.3211433628482485,
+      "learning_rate": 6.629541946462816e-06,
+      "loss": 0.8527,
+      "step": 165
+    },
+    {
+      "epoch": 4.15,
+      "grad_norm": 0.3547685084940225,
+      "learning_rate": 6.591214903137221e-06,
+      "loss": 0.8621,
+      "step": 166
+    },
+    {
+      "epoch": 4.175,
+      "grad_norm": 0.33781250869441176,
+      "learning_rate": 6.552783540969212e-06,
+      "loss": 0.8455,
+      "step": 167
+    },
+    {
+      "epoch": 4.2,
+      "grad_norm": 0.30993567341499995,
+      "learning_rate": 6.514250379489754e-06,
+      "loss": 0.8417,
+      "step": 168
+    },
+    {
+      "epoch": 4.225,
+      "grad_norm": 0.33582932826059514,
+      "learning_rate": 6.475617944903693e-06,
+      "loss": 0.8326,
+      "step": 169
+    },
+    {
+      "epoch": 4.25,
+      "grad_norm": 0.46046634589220586,
+      "learning_rate": 6.436888769924142e-06,
+      "loss": 0.8663,
+      "step": 170
+    },
+    {
+      "epoch": 4.275,
+      "grad_norm": 0.32228319665068555,
+      "learning_rate": 6.398065393606445e-06,
+      "loss": 0.8405,
+      "step": 171
+    },
+    {
+      "epoch": 4.3,
+      "grad_norm": 0.36506022581982417,
+      "learning_rate": 6.3591503611817155e-06,
+      "loss": 0.8633,
+      "step": 172
+    },
+    {
+      "epoch": 4.325,
+      "grad_norm": 0.35728088386983137,
+      "learning_rate": 6.320146223889966e-06,
+      "loss": 0.8451,
+      "step": 173
+    },
+    {
+      "epoch": 4.35,
+      "grad_norm": 0.3389049900239143,
+      "learning_rate": 6.281055538812861e-06,
+      "loss": 0.8383,
+      "step": 174
+    },
+    {
+      "epoch": 4.375,
+      "grad_norm": 0.35164642827686565,
+      "learning_rate": 6.241880868706075e-06,
+      "loss": 0.8436,
+      "step": 175
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 0.45602132072592544,
+      "learning_rate": 6.202624781831269e-06,
+      "loss": 0.8234,
+      "step": 176
+    },
+    {
+      "epoch": 4.425,
+      "grad_norm": 0.31307814205851997,
+      "learning_rate": 6.163289851787732e-06,
+      "loss": 0.8383,
+      "step": 177
+    },
+    {
+      "epoch": 4.45,
+      "grad_norm": 0.3443353966575101,
+      "learning_rate": 6.123878657343648e-06,
+      "loss": 0.8057,
+      "step": 178
+    },
+    {
+      "epoch": 4.475,
+      "grad_norm": 0.33895295083957905,
+      "learning_rate": 6.084393782267039e-06,
+      "loss": 0.8404,
+      "step": 179
+    },
+    {
+      "epoch": 4.5,
+      "grad_norm": 0.35609060580352403,
+      "learning_rate": 6.044837815156377e-06,
+      "loss": 0.8219,
+      "step": 180
+    },
+    {
+      "epoch": 4.525,
+      "grad_norm": 0.34688264086763054,
+      "learning_rate": 6.005213349270864e-06,
+      "loss": 0.8244,
+      "step": 181
+    },
+    {
+      "epoch": 4.55,
+      "grad_norm": 0.34841190032285607,
+      "learning_rate": 5.965522982360441e-06,
+      "loss": 0.827,
+      "step": 182
+    },
+    {
+      "epoch": 4.575,
+      "grad_norm": 0.41157723970826277,
+      "learning_rate": 5.925769316495461e-06,
+      "loss": 0.8233,
+      "step": 183
+    },
+    {
+      "epoch": 4.6,
+      "grad_norm": 0.2871326799596576,
+      "learning_rate": 5.885954957896115e-06,
+      "loss": 0.8096,
+      "step": 184
+    },
+    {
+      "epoch": 4.625,
+      "grad_norm": 0.35012249585775307,
+      "learning_rate": 5.846082516761558e-06,
+      "loss": 0.7931,
+      "step": 185
+    },
+    {
+      "epoch": 4.65,
+      "grad_norm": 0.46359796922380764,
+      "learning_rate": 5.806154607098799e-06,
+      "loss": 0.8283,
+      "step": 186
+    },
+    {
+      "epoch": 4.675,
+      "grad_norm": 0.4258979061317278,
+      "learning_rate": 5.766173846551316e-06,
+      "loss": 0.8174,
+      "step": 187
+    },
+    {
+      "epoch": 4.7,
+      "grad_norm": 0.3462495330359986,
+      "learning_rate": 5.726142856227453e-06,
+      "loss": 0.8023,
+      "step": 188
+    },
+    {
+      "epoch": 4.725,
+      "grad_norm": 0.38828158563310117,
+      "learning_rate": 5.686064260528577e-06,
+      "loss": 0.8297,
+      "step": 189
+    },
+    {
+      "epoch": 4.75,
+      "grad_norm": 0.33951477432576194,
+      "learning_rate": 5.645940686977033e-06,
+      "loss": 0.8246,
+      "step": 190
+    },
+    {
+      "epoch": 4.775,
+      "grad_norm": 0.3095495826095541,
+      "learning_rate": 5.6057747660438745e-06,
+      "loss": 0.8129,
+      "step": 191
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 0.4054254828876582,
+      "learning_rate": 5.5655691309764225e-06,
+      "loss": 0.8313,
+      "step": 192
+    },
+    {
+      "epoch": 4.825,
+      "grad_norm": 0.3483399073445393,
+      "learning_rate": 5.52532641762562e-06,
+      "loss": 0.8129,
+      "step": 193
+    },
+    {
+      "epoch": 4.85,
+      "grad_norm": 0.32176037839806315,
+      "learning_rate": 5.485049264273241e-06,
+      "loss": 0.7842,
+      "step": 194
+    },
+    {
+      "epoch": 4.875,
+      "grad_norm": 0.29495279378063277,
+      "learning_rate": 5.444740311458915e-06,
+      "loss": 0.7841,
+      "step": 195
+    },
+    {
+      "epoch": 4.9,
+      "grad_norm": 0.3484345853234177,
+      "learning_rate": 5.404402201807022e-06,
+      "loss": 0.8362,
+      "step": 196
+    },
+    {
+      "epoch": 4.925,
+      "grad_norm": 0.32603753016594705,
+      "learning_rate": 5.364037579853441e-06,
+      "loss": 0.8055,
+      "step": 197
+    },
+    {
+      "epoch": 4.95,
+      "grad_norm": 0.34113760948421906,
+      "learning_rate": 5.323649091872179e-06,
+      "loss": 0.7877,
+      "step": 198
+    },
+    {
+      "epoch": 4.975,
+      "grad_norm": 0.33921943319034875,
+      "learning_rate": 5.283239385701882e-06,
+      "loss": 0.805,
+      "step": 199
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 0.2951107290602008,
+      "learning_rate": 5.242811110572243e-06,
+      "loss": 0.7991,
+      "step": 200
+    },
+    {
+      "epoch": 5.025,
+      "grad_norm": 0.30253566064261156,
+      "learning_rate": 5.202366916930319e-06,
+      "loss": 0.8044,
+      "step": 201
+    },
+    {
+      "epoch": 5.05,
+      "grad_norm": 0.3230565909231367,
+      "learning_rate": 5.161909456266781e-06,
+      "loss": 0.7646,
+      "step": 202
+    },
+    {
+      "epoch": 5.075,
+      "grad_norm": 0.3180667356505713,
+      "learning_rate": 5.121441380942066e-06,
+      "loss": 0.8159,
+      "step": 203
+    },
+    {
+      "epoch": 5.1,
+      "grad_norm": 0.36374034176814063,
+      "learning_rate": 5.080965344012509e-06,
+      "loss": 0.7713,
+      "step": 204
+    },
+    {
+      "epoch": 5.125,
+      "grad_norm": 0.30290892981437073,
+      "learning_rate": 5.040483999056393e-06,
+      "loss": 0.767,
+      "step": 205
+    },
+    {
+      "epoch": 5.15,
+      "grad_norm": 0.3930116162781173,
+      "learning_rate": 5e-06,
+      "loss": 0.7916,
+      "step": 206
+    },
+    {
+      "epoch": 5.175,
+      "grad_norm": 0.3185992766019151,
+      "learning_rate": 4.9595160009436075e-06,
+      "loss": 0.7996,
+      "step": 207
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 0.38832542048067603,
+      "learning_rate": 4.919034655987493e-06,
+      "loss": 0.7878,
+      "step": 208
+    },
+    {
+      "epoch": 5.225,
+      "grad_norm": 0.39110099000218684,
+      "learning_rate": 4.8785586190579355e-06,
+      "loss": 0.7728,
+      "step": 209
+    },
+    {
+      "epoch": 5.25,
+      "grad_norm": 0.30581266996926304,
+      "learning_rate": 4.838090543733222e-06,
+      "loss": 0.7937,
+      "step": 210
+    },
+    {
+      "epoch": 5.275,
+      "grad_norm": 0.31131833564138595,
+      "learning_rate": 4.797633083069683e-06,
+      "loss": 0.758,
+      "step": 211
+    },
+    {
+      "epoch": 5.3,
+      "grad_norm": 0.3726348002148966,
+      "learning_rate": 4.757188889427761e-06,
+      "loss": 0.8192,
+      "step": 212
+    },
+    {
+      "epoch": 5.325,
+      "grad_norm": 0.32124465048482537,
+      "learning_rate": 4.716760614298118e-06,
+      "loss": 0.8127,
+      "step": 213
+    },
+    {
+      "epoch": 5.35,
+      "grad_norm": 0.4291943729340545,
+      "learning_rate": 4.6763509081278215e-06,
+      "loss": 0.7956,
+      "step": 214
+    },
+    {
+      "epoch": 5.375,
+      "grad_norm": 0.28810585263359334,
+      "learning_rate": 4.63596242014656e-06,
+      "loss": 0.7876,
+      "step": 215
+    },
+    {
+      "epoch": 5.4,
+      "grad_norm": 0.36635836033703345,
+      "learning_rate": 4.59559779819298e-06,
+      "loss": 0.8118,
+      "step": 216
+    },
+    {
+      "epoch": 5.425,
+      "grad_norm": 0.3989844143804462,
+      "learning_rate": 4.555259688541087e-06,
+      "loss": 0.7744,
+      "step": 217
+    },
+    {
+      "epoch": 5.45,
+      "grad_norm": 0.3775475757163417,
+      "learning_rate": 4.51495073572676e-06,
+      "loss": 0.8076,
+      "step": 218
+    },
+    {
+      "epoch": 5.475,
+      "grad_norm": 0.3885576211506495,
+      "learning_rate": 4.474673582374381e-06,
+      "loss": 0.8152,
+      "step": 219
+    },
+    {
+      "epoch": 5.5,
+      "grad_norm": 0.5486181305427187,
+      "learning_rate": 4.434430869023579e-06,
+      "loss": 0.8035,
+      "step": 220
+    },
+    {
+      "epoch": 5.525,
+      "grad_norm": 0.449133416014458,
+      "learning_rate": 4.394225233956127e-06,
+      "loss": 0.7581,
+      "step": 221
+    },
+    {
+      "epoch": 5.55,
+      "grad_norm": 0.3532754327449149,
+      "learning_rate": 4.3540593130229695e-06,
+      "loss": 0.8145,
+      "step": 222
+    },
+    {
+      "epoch": 5.575,
+      "grad_norm": 0.6472978242633896,
+      "learning_rate": 4.313935739471425e-06,
+      "loss": 0.7889,
+      "step": 223
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 0.5593763414902034,
+      "learning_rate": 4.27385714377255e-06,
+      "loss": 0.8025,
+      "step": 224
+    },
+    {
+      "epoch": 5.625,
+      "grad_norm": 0.3113555109979358,
+      "learning_rate": 4.233826153448685e-06,
+      "loss": 0.8235,
+      "step": 225
+    },
+    {
+      "epoch": 5.65,
+      "grad_norm": 0.32783417993682323,
+      "learning_rate": 4.1938453929012014e-06,
+      "loss": 0.7839,
+      "step": 226
+    },
+    {
+      "epoch": 5.675,
+      "grad_norm": 0.47735491344985437,
+      "learning_rate": 4.1539174832384425e-06,
+      "loss": 0.7673,
+      "step": 227
+    },
+    {
+      "epoch": 5.7,
+      "grad_norm": 0.379167946256265,
+      "learning_rate": 4.1140450421038865e-06,
+      "loss": 0.7669,
+      "step": 228
+    },
+    {
+      "epoch": 5.725,
+      "grad_norm": 0.4055179197901725,
+      "learning_rate": 4.074230683504541e-06,
+      "loss": 0.7915,
+      "step": 229
+    },
+    {
+      "epoch": 5.75,
+      "grad_norm": 0.33990082918663317,
+      "learning_rate": 4.034477017639561e-06,
+      "loss": 0.7753,
+      "step": 230
+    },
+    {
+      "epoch": 5.775,
+      "grad_norm": 0.3440010530347358,
+      "learning_rate": 3.994786650729137e-06,
+      "loss": 0.7834,
+      "step": 231
+    },
+    {
+      "epoch": 5.8,
+      "grad_norm": 0.4571564594873592,
+      "learning_rate": 3.955162184843625e-06,
+      "loss": 0.7843,
+      "step": 232
+    },
+    {
+      "epoch": 5.825,
+      "grad_norm": 0.31044612074261246,
+      "learning_rate": 3.9156062177329624e-06,
+      "loss": 0.7821,
+      "step": 233
+    },
+    {
+      "epoch": 5.85,
+      "grad_norm": 0.31765843548865946,
+      "learning_rate": 3.8761213426563546e-06,
+      "loss": 0.7648,
+      "step": 234
+    },
+    {
+      "epoch": 5.875,
+      "grad_norm": 0.33542686760185636,
+      "learning_rate": 3.836710148212271e-06,
+      "loss": 0.7653,
+      "step": 235
+    },
+    {
+      "epoch": 5.9,
+      "grad_norm": 0.34867235131286023,
+      "learning_rate": 3.7973752181687336e-06,
+      "loss": 0.7818,
+      "step": 236
+    },
+    {
+      "epoch": 5.925,
+      "grad_norm": 0.42551356848317706,
+      "learning_rate": 3.7581191312939256e-06,
+      "loss": 0.791,
+      "step": 237
+    },
+    {
+      "epoch": 5.95,
+      "grad_norm": 0.30823548064435274,
+      "learning_rate": 3.7189444611871383e-06,
+      "loss": 0.7892,
+      "step": 238
+    },
+    {
+      "epoch": 5.975,
+      "grad_norm": 0.343637195813578,
+      "learning_rate": 3.679853776110035e-06,
+      "loss": 0.763,
+      "step": 239
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 0.33568955543808604,
+      "learning_rate": 3.6408496388182857e-06,
+      "loss": 0.7701,
+      "step": 240
+    },
+    {
+      "epoch": 6.025,
+      "grad_norm": 0.39497924967421005,
+      "learning_rate": 3.601934606393556e-06,
+      "loss": 0.7668,
+      "step": 241
+    },
+    {
+      "epoch": 6.05,
+      "grad_norm": 0.3461417085165549,
+      "learning_rate": 3.5631112300758595e-06,
+      "loss": 0.7444,
+      "step": 242
+    },
+    {
+      "epoch": 6.075,
+      "grad_norm": 0.32325669130792,
+      "learning_rate": 3.524382055096309e-06,
+      "loss": 0.7814,
+      "step": 243
+    },
+    {
+      "epoch": 6.1,
+      "grad_norm": 0.294314054885741,
+      "learning_rate": 3.4857496205102475e-06,
+      "loss": 0.7996,
+      "step": 244
+    },
+    {
+      "epoch": 6.125,
+      "grad_norm": 0.30410205632168436,
+      "learning_rate": 3.4472164590307895e-06,
+      "loss": 0.7741,
+      "step": 245
+    },
+    {
+      "epoch": 6.15,
+      "grad_norm": 0.3747577565198413,
+      "learning_rate": 3.4087850968627823e-06,
+      "loss": 0.7651,
+      "step": 246
+    },
+    {
+      "epoch": 6.175,
+      "grad_norm": 0.3480834902968779,
+      "learning_rate": 3.370458053537186e-06,
+      "loss": 0.7728,
+      "step": 247
+    },
+    {
+      "epoch": 6.2,
+      "grad_norm": 0.30874816092153307,
+      "learning_rate": 3.3322378417458985e-06,
+      "loss": 0.7667,
+      "step": 248
+    },
+    {
+      "epoch": 6.225,
+      "grad_norm": 0.3220383343921079,
+      "learning_rate": 3.2941269671770193e-06,
+      "loss": 0.7962,
+      "step": 249
+    },
+    {
+      "epoch": 6.25,
+      "grad_norm": 0.3535972366319736,
+      "learning_rate": 3.2561279283505888e-06,
+      "loss": 0.7875,
+      "step": 250
+    },
+    {
+      "epoch": 6.275,
+      "grad_norm": 0.3570142484942382,
+      "learning_rate": 3.218243216454775e-06,
+      "loss": 0.7459,
+      "step": 251
+    },
+    {
+      "epoch": 6.3,
+      "grad_norm": 0.3938420232838248,
+      "learning_rate": 3.180475315182563e-06,
+      "loss": 0.7866,
+      "step": 252
+    },
+    {
+      "epoch": 6.325,
+      "grad_norm": 0.418670895094041,
+      "learning_rate": 3.1428267005689182e-06,
+      "loss": 0.7465,
+      "step": 253
+    },
+    {
+      "epoch": 6.35,
+      "grad_norm": 0.4361674812154792,
+      "learning_rate": 3.1052998408284664e-06,
+      "loss": 0.7823,
+      "step": 254
+    },
+    {
+      "epoch": 6.375,
+      "grad_norm": 0.5410687457707055,
+      "learning_rate": 3.0678971961936765e-06,
+      "loss": 0.7888,
+      "step": 255
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 0.3881527874591265,
+      "learning_rate": 3.0306212187535653e-06,
+      "loss": 0.7975,
+      "step": 256
+    },
+    {
+      "epoch": 6.425,
+      "grad_norm": 0.3540262179710572,
+      "learning_rate": 2.9934743522929476e-06,
+      "loss": 0.8021,
+      "step": 257
+    },
+    {
+      "epoch": 6.45,
+      "grad_norm": 0.41920366313347873,
+      "learning_rate": 2.9564590321322206e-06,
+      "loss": 0.7383,
+      "step": 258
+    },
+    {
+      "epoch": 6.475,
+      "grad_norm": 0.4628906287676455,
+      "learning_rate": 2.9195776849677037e-06,
+      "loss": 0.7476,
+      "step": 259
+    },
+    {
+      "epoch": 6.5,
+      "grad_norm": 0.44302784747531426,
+      "learning_rate": 2.882832728712551e-06,
+      "loss": 0.7715,
+      "step": 260
+    },
+    {
+      "epoch": 6.525,
+      "grad_norm": 0.35338224459584966,
+      "learning_rate": 2.8462265723382255e-06,
+      "loss": 0.7417,
+      "step": 261
+    },
+    {
+      "epoch": 6.55,
+      "grad_norm": 0.48937540088183007,
+      "learning_rate": 2.8097616157165886e-06,
+      "loss": 0.7533,
+      "step": 262
+    },
+    {
+      "epoch": 6.575,
+      "grad_norm": 0.45167389676949854,
+      "learning_rate": 2.7734402494625395e-06,
+      "loss": 0.764,
+      "step": 263
+    },
+    {
+      "epoch": 6.6,
+      "grad_norm": 0.40438810739435926,
+      "learning_rate": 2.7372648547773063e-06,
+      "loss": 0.7963,
+      "step": 264
+    },
+    {
+      "epoch": 6.625,
+      "grad_norm": 0.3424135964248357,
+      "learning_rate": 2.701237803292335e-06,
+      "loss": 0.7782,
+      "step": 265
+    },
+    {
+      "epoch": 6.65,
+      "grad_norm": 0.37838480593841406,
+      "learning_rate": 2.665361456913797e-06,
+      "loss": 0.7717,
+      "step": 266
+    },
+    {
+      "epoch": 6.675,
+      "grad_norm": 0.39264044408814786,
+      "learning_rate": 2.629638167667761e-06,
+      "loss": 0.7466,
+      "step": 267
+    },
+    {
+      "epoch": 6.7,
+      "grad_norm": 0.5528131987506779,
+      "learning_rate": 2.594070277545975e-06,
+      "loss": 0.7671,
+      "step": 268
+    },
+    {
+      "epoch": 6.725,
+      "grad_norm": 0.4036506169957066,
+      "learning_rate": 2.5586601183523485e-06,
+      "loss": 0.7525,
+      "step": 269
+    },
+    {
+      "epoch": 6.75,
+      "grad_norm": 0.30570416336137085,
+      "learning_rate": 2.5234100115500643e-06,
+      "loss": 0.7467,
+      "step": 270
+    },
+    {
+      "epoch": 6.775,
+      "grad_norm": 0.3985125428120007,
+      "learning_rate": 2.4883222681093916e-06,
+      "loss": 0.7971,
+      "step": 271
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 0.36468716140455315,
+      "learning_rate": 2.4533991883561868e-06,
+      "loss": 0.7687,
+      "step": 272
+    },
+    {
+      "epoch": 6.825,
+      "grad_norm": 0.3198173933913146,
+      "learning_rate": 2.4186430618210704e-06,
+      "loss": 0.7414,
+      "step": 273
+    },
+    {
+      "epoch": 6.85,
+      "grad_norm": 0.3303319933176365,
+      "learning_rate": 2.38405616708935e-06,
+      "loss": 0.7408,
+      "step": 274
+    },
+    {
+      "epoch": 6.875,
+      "grad_norm": 0.3258403268612293,
+      "learning_rate": 2.349640771651611e-06,
+      "loss": 0.7456,
+      "step": 275
+    },
+    {
+      "epoch": 6.9,
+      "grad_norm": 0.2812707026342879,
+      "learning_rate": 2.315399131755081e-06,
+      "loss": 0.7492,
+      "step": 276
+    },
+    {
+      "epoch": 6.925,
+      "grad_norm": 0.3177919316007201,
+      "learning_rate": 2.281333492255708e-06,
+      "loss": 0.7887,
+      "step": 277
+    },
+    {
+      "epoch": 6.95,
+      "grad_norm": 0.391540398746241,
+      "learning_rate": 2.2474460864709825e-06,
+      "loss": 0.7346,
+      "step": 278
+    },
+    {
+      "epoch": 6.975,
+      "grad_norm": 0.33491616417366366,
+      "learning_rate": 2.213739136033533e-06,
+      "loss": 0.7574,
+      "step": 279
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 0.31756042780759597,
+      "learning_rate": 2.1802148507454675e-06,
+      "loss": 0.7495,
+      "step": 280
+    },
+    {
+      "epoch": 7.025,
+      "grad_norm": 0.31406443802012124,
+      "learning_rate": 2.1468754284335097e-06,
+      "loss": 0.7587,
+      "step": 281
+    },
+    {
+      "epoch": 7.05,
+      "grad_norm": 0.3013777061132176,
+      "learning_rate": 2.1137230548049042e-06,
+      "loss": 0.7607,
+      "step": 282
+    },
+    {
+      "epoch": 7.075,
+      "grad_norm": 0.31696862178258495,
+      "learning_rate": 2.0807599033041238e-06,
+      "loss": 0.7728,
+      "step": 283
+    },
+    {
+      "epoch": 7.1,
+      "grad_norm": 0.36781880744277745,
+      "learning_rate": 2.0479881349703885e-06,
+      "loss": 0.7648,
+      "step": 284
+    },
+    {
+      "epoch": 7.125,
+      "grad_norm": 0.3229421517215962,
+      "learning_rate": 2.015409898295975e-06,
+      "loss": 0.7593,
+      "step": 285
+    },
+    {
+      "epoch": 7.15,
+      "grad_norm": 0.3567808682666007,
+      "learning_rate": 1.983027329085377e-06,
+      "loss": 0.7784,
+      "step": 286
+    },
+    {
+      "epoch": 7.175,
+      "grad_norm": 0.3523029725508882,
+      "learning_rate": 1.950842550315277e-06,
+      "loss": 0.7428,
+      "step": 287
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 0.3433500921490076,
+      "learning_rate": 1.9188576719953635e-06,
+      "loss": 0.7505,
+      "step": 288
+    },
+    {
+      "epoch": 7.225,
+      "grad_norm": 0.31594045765829454,
+      "learning_rate": 1.8870747910300064e-06,
+      "loss": 0.7618,
+      "step": 289
+    },
+    {
+      "epoch": 7.25,
+      "grad_norm": 0.2848499737108094,
+      "learning_rate": 1.8554959910807773e-06,
+      "loss": 0.7421,
+      "step": 290
+    },
+    {
+      "epoch": 7.275,
+      "grad_norm": 0.3607308089002104,
+      "learning_rate": 1.8241233424298582e-06,
+      "loss": 0.7641,
+      "step": 291
+    },
+    {
+      "epoch": 7.3,
+      "grad_norm": 0.36528202865847964,
+      "learning_rate": 1.7929589018443016e-06,
+      "loss": 0.7493,
+      "step": 292
+    },
+    {
+      "epoch": 7.325,
+      "grad_norm": 0.3309081277389018,
+      "learning_rate": 1.7620047124412e-06,
+      "loss": 0.7395,
+      "step": 293
+    },
+    {
+      "epoch": 7.35,
+      "grad_norm": 0.31413373720883053,
+      "learning_rate": 1.7312628035537388e-06,
+      "loss": 0.771,
+      "step": 294
+    },
+    {
+      "epoch": 7.375,
+      "grad_norm": 0.3095504162989951,
+      "learning_rate": 1.7007351905981512e-06,
+      "loss": 0.7466,
+      "step": 295
+    },
+    {
+      "epoch": 7.4,
+      "grad_norm": 0.307807059177367,
+      "learning_rate": 1.6704238749415958e-06,
+      "loss": 0.761,
+      "step": 296
+    },
+    {
+      "epoch": 7.425,
+      "grad_norm": 0.31572298722999037,
+      "learning_rate": 1.640330843770938e-06,
+      "loss": 0.7558,
+      "step": 297
+    },
+    {
+      "epoch": 7.45,
+      "grad_norm": 0.3419550050125546,
+      "learning_rate": 1.6104580699624839e-06,
+      "loss": 0.7641,
+      "step": 298
+    },
+    {
+      "epoch": 7.475,
+      "grad_norm": 0.32031695230802487,
+      "learning_rate": 1.5808075119526323e-06,
+      "loss": 0.7527,
+      "step": 299
+    },
+    {
+      "epoch": 7.5,
+      "grad_norm": 0.29808690522996345,
+      "learning_rate": 1.5513811136094786e-06,
+      "loss": 0.7611,
+      "step": 300
+    },
+    {
+      "epoch": 7.525,
+      "grad_norm": 0.36134876879819094,
+      "learning_rate": 1.5221808041053877e-06,
+      "loss": 0.7274,
+      "step": 301
+    },
+    {
+      "epoch": 7.55,
+      "grad_norm": 0.3297181285038958,
+      "learning_rate": 1.4932084977905043e-06,
+      "loss": 0.7524,
+      "step": 302
+    },
+    {
+      "epoch": 7.575,
+      "grad_norm": 0.31834816355878326,
+      "learning_rate": 1.4644660940672628e-06,
+      "loss": 0.7435,
+      "step": 303
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 0.42757726677974806,
+      "learning_rate": 1.4359554772658551e-06,
+      "loss": 0.7623,
+      "step": 304
+    },
+    {
+      "epoch": 7.625,
+      "grad_norm": 0.28078915653003605,
+      "learning_rate": 1.4076785165206962e-06,
+      "loss": 0.7468,
+      "step": 305
+    },
+    {
+      "epoch": 7.65,
+      "grad_norm": 0.3451494754079205,
+      "learning_rate": 1.3796370656478936e-06,
+      "loss": 0.7655,
+      "step": 306
+    },
+    {
+      "epoch": 7.675,
+      "grad_norm": 0.41617962192047586,
+      "learning_rate": 1.3518329630236987e-06,
+      "loss": 0.7445,
+      "step": 307
+    },
+    {
+      "epoch": 7.7,
+      "grad_norm": 0.43512318327984323,
+      "learning_rate": 1.3242680314639995e-06,
+      "loss": 0.764,
+      "step": 308
+    },
+    {
+      "epoch": 7.725,
+      "grad_norm": 0.31431480720335797,
+      "learning_rate": 1.2969440781048015e-06,
+      "loss": 0.7738,
+      "step": 309
+    },
+    {
+      "epoch": 7.75,
+      "grad_norm": 0.31310147613394657,
+      "learning_rate": 1.2698628942837698e-06,
+      "loss": 0.7622,
+      "step": 310
+    },
+    {
+      "epoch": 7.775,
+      "grad_norm": 0.3318795115634929,
+      "learning_rate": 1.24302625542278e-06,
+      "loss": 0.7748,
+      "step": 311
+    },
+    {
+      "epoch": 7.8,
+      "grad_norm": 0.3562505259439764,
+      "learning_rate": 1.2164359209115235e-06,
+      "loss": 0.7576,
+      "step": 312
+    },
+    {
+      "epoch": 7.825,
+      "grad_norm": 0.29514903206992193,
+      "learning_rate": 1.1900936339921692e-06,
+      "loss": 0.7339,
+      "step": 313
+    },
+    {
+      "epoch": 7.85,
+      "grad_norm": 0.4232187961023811,
+      "learning_rate": 1.164001121645069e-06,
+      "loss": 0.794,
+      "step": 314
+    },
+    {
+      "epoch": 7.875,
+      "grad_norm": 0.3352202322909541,
+      "learning_rate": 1.1381600944755493e-06,
+      "loss": 0.7344,
+      "step": 315
+    },
+    {
+      "epoch": 7.9,
+      "grad_norm": 0.3402663403232913,
+      "learning_rate": 1.1125722466017547e-06,
+      "loss": 0.7658,
+      "step": 316
+    },
+    {
+      "epoch": 7.925,
+      "grad_norm": 0.3755980775420333,
+      "learning_rate": 1.0872392555435857e-06,
+      "loss": 0.7525,
+      "step": 317
+    },
+    {
+      "epoch": 7.95,
+      "grad_norm": 0.2938760848079364,
+      "learning_rate": 1.062162782112729e-06,
+      "loss": 0.7214,
+      "step": 318
+    },
+    {
+      "epoch": 7.975,
+      "grad_norm": 0.42894033568357076,
+      "learning_rate": 1.0373444703037644e-06,
+      "loss": 0.7467,
+      "step": 319
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 0.3310479508484881,
+      "learning_rate": 1.012785947186397e-06,
+      "loss": 0.7484,
+      "step": 320
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 200.0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 364787589054464.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

7b_cold_start_fixUnderfitting/checkpoint-epoch8/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33b9da033fae7acbc9187de817ead528cae0dcc901542c9cd55557fafb0b9a8e
+size 7800

7b_cold_start_fixUnderfitting/checkpoint-epoch8/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)