Upload folder using huggingface_hub

.gitattributes

@@ -56,3 +56,19 @@ checkpoint/vln/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_202
 checkpoint/vln/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_161003/checkpoint-4500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint/vln/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260228_043722/checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoint/vln/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260228_043722/checkpoint-5500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-3500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-4000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-4500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_160647/checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_160647/checkpoint-5500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_234733/checkpoint-6000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_234733/checkpoint-6500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260227_234733/checkpoint-7000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260228_091109/checkpoint-7500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260228_133506/checkpoint-8000/tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b32932de8927c14cdc81488ab0962904039e611adaa9df244324e2f6938cfbd5
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc068cd9fed6aa3459bd78ad1c84b10a7704fb9c46e36a6324dbf53f76cbdaa
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd0782c8356c8fb1e10267ac88d4df02ef08d5a17388f845261227b9ffa8abd
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3308981b0cfaad584b661e86709b0770ceb4f19aa164e50916197adb633996ff
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a5d15a7a33653dcb6410a06a7358a50ff710147c20d151c4687f412baf35ca7
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd1eb6dd962c8e1a0af6c5992cb6de7a9098690f2cfc9b7e733a5a69c92cb3a8
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1112f2895e175552e4dc362ae34bd8b255aa4869e4c34dff8ad27d6b9b72014
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9467e953079a33b38ee5cfe74e5397f0e266954a0c2a577ebc02fcb539e89a
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af6c3e37c6db2f23310108dbd727469e0dfc74e6918284ae072c50d91d20daa1
+size 940304657

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a51a412f25e7027edf8c6421dec1365f13892022704c79d4e442c1408df33dc
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7867f6e00af0e47667894620a6167524de5fb53283d81030ab4a3299b8d511c
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:106c3a0fdcd74190ee6601d52c9df5dbbc8b75d3a8cd9072111c2ad90d1dba90
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:989058e08353cef0f9188495d8fb4d611b9fc83c7bf585b7d98312fbe757f7c7
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:013c28b516232514e84534bbc97f004cb2c70e2ca9c6363150f80f6e6f8ebf5c
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7720498ed794dec47cec3b9d6c4d95880502b69cbc8264769041286e911f54b
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/global_step500/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72fa1c79ed2020f74b0faea2d048db09ba9609d0e6ee650c076b1340789a8ff0
+size 940304529

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/latest

@@ -0,0 +1 @@
+global_step500

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc35f93d9fd72a4e9ea5e37ce95bfbc15a505a1c34fc8b329416d60f3f0a6a2c
+size 7968858171

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6534b27f9f6a92a1c33c78debd97b09677197e112a51264426e5bdf3c941ae02
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5964090bcdfbd7bd81be11d75b7e50ccbe358b5e1948b3d1a2b0d622ac55602
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd764d57589936954d768de57e756633e29ffc70761c5479e06664ae7aaa8195
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6263b2417796e3e98654df07230818076ae31abfef88bed6b0eeb1f5d798389c
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e3e1c9d8ccc96a70e2ffa1f277757cf67f00ab388670b58ab2952d6656ff2b
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0abdf7e3397bf822a5cb49cc333782ae711c339a6f0fdc4ed5b28438f40423a8
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d750f9af677b6b44bbbd2b6943cd871ece263697e4aa348e7def7b05fe6640
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c0ec9e6d32cf4a0bf99aecad877b0fc914dae47eeef120a1e2762399ab3ef32
+size 16389

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a03fb5adda7d09f72a4631642ef8adfab631c21735720c45fd9fa91cc06ffc62
+size 1465

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/trainer_state.json

@@ -0,0 +1,3542 @@
+{
+  "best_global_step": 500,
+  "best_metric": 0.11392344534397125,
+  "best_model_checkpoint": "/mnt/dolphinfs/ssd_pool/docker/user/hadoop-aipnlp/EVA/yangheqing/workspace/Streaming-UAV/checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500",
+  "epoch": 1.5625,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003125,
+      "grad_norm": 25.967828613892614,
+      "learning_rate": 0.0,
+      "loss": 6.5586,
+      "step": 1
+    },
+    {
+      "epoch": 0.00625,
+      "grad_norm": 25.27369746937231,
+      "learning_rate": 5.208333333333334e-07,
+      "loss": 6.5314,
+      "step": 2
+    },
+    {
+      "epoch": 0.009375,
+      "grad_norm": 25.95039198787132,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 6.5288,
+      "step": 3
+    },
+    {
+      "epoch": 0.0125,
+      "grad_norm": 25.986133150119834,
+      "learning_rate": 1.5625e-06,
+      "loss": 6.5212,
+      "step": 4
+    },
+    {
+      "epoch": 0.015625,
+      "grad_norm": 25.35735654276883,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 6.5212,
+      "step": 5
+    },
+    {
+      "epoch": 0.01875,
+      "grad_norm": 25.940538420900417,
+      "learning_rate": 2.604166666666667e-06,
+      "loss": 6.5215,
+      "step": 6
+    },
+    {
+      "epoch": 0.021875,
+      "grad_norm": 24.463599851871844,
+      "learning_rate": 3.125e-06,
+      "loss": 6.355,
+      "step": 7
+    },
+    {
+      "epoch": 0.025,
+      "grad_norm": 24.678097351395774,
+      "learning_rate": 3.6458333333333337e-06,
+      "loss": 6.3536,
+      "step": 8
+    },
+    {
+      "epoch": 0.028125,
+      "grad_norm": 23.289714124306812,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 6.2804,
+      "step": 9
+    },
+    {
+      "epoch": 0.03125,
+      "grad_norm": 18.124421370186344,
+      "learning_rate": 4.6875000000000004e-06,
+      "loss": 5.9191,
+      "step": 10
+    },
+    {
+      "epoch": 0.034375,
+      "grad_norm": 16.525693598489248,
+      "learning_rate": 5.208333333333334e-06,
+      "loss": 5.7543,
+      "step": 11
+    },
+    {
+      "epoch": 0.0375,
+      "grad_norm": 16.042242903864597,
+      "learning_rate": 5.729166666666667e-06,
+      "loss": 5.7337,
+      "step": 12
+    },
+    {
+      "epoch": 0.040625,
+      "grad_norm": 3.1780981319773716,
+      "learning_rate": 6.25e-06,
+      "loss": 5.3001,
+      "step": 13
+    },
+    {
+      "epoch": 0.04375,
+      "grad_norm": 3.2593658887720296,
+      "learning_rate": 6.770833333333333e-06,
+      "loss": 5.2489,
+      "step": 14
+    },
+    {
+      "epoch": 0.046875,
+      "grad_norm": 3.1954466175424807,
+      "learning_rate": 7.2916666666666674e-06,
+      "loss": 5.1933,
+      "step": 15
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 4.090039756235578,
+      "learning_rate": 7.8125e-06,
+      "loss": 5.1982,
+      "step": 16
+    },
+    {
+      "epoch": 0.053125,
+      "grad_norm": 5.140187486881889,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 5.0604,
+      "step": 17
+    },
+    {
+      "epoch": 0.05625,
+      "grad_norm": 9.443764678804124,
+      "learning_rate": 8.854166666666667e-06,
+      "loss": 5.09,
+      "step": 18
+    },
+    {
+      "epoch": 0.059375,
+      "grad_norm": 5.574995913524894,
+      "learning_rate": 9.375000000000001e-06,
+      "loss": 5.0904,
+      "step": 19
+    },
+    {
+      "epoch": 0.0625,
+      "grad_norm": 4.121387571654496,
+      "learning_rate": 9.895833333333333e-06,
+      "loss": 4.9674,
+      "step": 20
+    },
+    {
+      "epoch": 0.065625,
+      "grad_norm": 4.823284029584046,
+      "learning_rate": 1.0416666666666668e-05,
+      "loss": 4.9051,
+      "step": 21
+    },
+    {
+      "epoch": 0.06875,
+      "grad_norm": 6.388610275286726,
+      "learning_rate": 1.09375e-05,
+      "loss": 4.8703,
+      "step": 22
+    },
+    {
+      "epoch": 0.071875,
+      "grad_norm": 6.854783102694158,
+      "learning_rate": 1.1458333333333333e-05,
+      "loss": 4.7715,
+      "step": 23
+    },
+    {
+      "epoch": 0.075,
+      "grad_norm": 6.197655397763414,
+      "learning_rate": 1.1979166666666667e-05,
+      "loss": 4.6821,
+      "step": 24
+    },
+    {
+      "epoch": 0.078125,
+      "grad_norm": 5.606416900952863,
+      "learning_rate": 1.25e-05,
+      "loss": 4.5755,
+      "step": 25
+    },
+    {
+      "epoch": 0.08125,
+      "grad_norm": 6.236956340270923,
+      "learning_rate": 1.3020833333333334e-05,
+      "loss": 4.3759,
+      "step": 26
+    },
+    {
+      "epoch": 0.084375,
+      "grad_norm": 6.197108972062877,
+      "learning_rate": 1.3541666666666666e-05,
+      "loss": 4.2675,
+      "step": 27
+    },
+    {
+      "epoch": 0.0875,
+      "grad_norm": 6.78633313102827,
+      "learning_rate": 1.4062500000000001e-05,
+      "loss": 4.0308,
+      "step": 28
+    },
+    {
+      "epoch": 0.090625,
+      "grad_norm": 7.686330442727373,
+      "learning_rate": 1.4583333333333335e-05,
+      "loss": 3.9698,
+      "step": 29
+    },
+    {
+      "epoch": 0.09375,
+      "grad_norm": 9.386600193133901,
+      "learning_rate": 1.5104166666666667e-05,
+      "loss": 3.8336,
+      "step": 30
+    },
+    {
+      "epoch": 0.096875,
+      "grad_norm": 6.488756092595361,
+      "learning_rate": 1.5625e-05,
+      "loss": 3.6063,
+      "step": 31
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 10.0362265096323,
+      "learning_rate": 1.6145833333333334e-05,
+      "loss": 3.5027,
+      "step": 32
+    },
+    {
+      "epoch": 0.103125,
+      "grad_norm": 15.647466852814294,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 3.4859,
+      "step": 33
+    },
+    {
+      "epoch": 0.10625,
+      "grad_norm": 11.855991684102108,
+      "learning_rate": 1.71875e-05,
+      "loss": 3.4019,
+      "step": 34
+    },
+    {
+      "epoch": 0.109375,
+      "grad_norm": 12.699059012381074,
+      "learning_rate": 1.7708333333333335e-05,
+      "loss": 3.309,
+      "step": 35
+    },
+    {
+      "epoch": 0.1125,
+      "grad_norm": 13.896904162434533,
+      "learning_rate": 1.8229166666666668e-05,
+      "loss": 3.3102,
+      "step": 36
+    },
+    {
+      "epoch": 0.115625,
+      "grad_norm": 11.951083780214015,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 3.2363,
+      "step": 37
+    },
+    {
+      "epoch": 0.11875,
+      "grad_norm": 11.10545801119009,
+      "learning_rate": 1.9270833333333335e-05,
+      "loss": 3.1295,
+      "step": 38
+    },
+    {
+      "epoch": 0.121875,
+      "grad_norm": 16.79721364934682,
+      "learning_rate": 1.9791666666666665e-05,
+      "loss": 3.0404,
+      "step": 39
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 12.149686062939905,
+      "learning_rate": 2.0312500000000002e-05,
+      "loss": 2.9296,
+      "step": 40
+    },
+    {
+      "epoch": 0.128125,
+      "grad_norm": 10.118283626422649,
+      "learning_rate": 2.0833333333333336e-05,
+      "loss": 3.0206,
+      "step": 41
+    },
+    {
+      "epoch": 0.13125,
+      "grad_norm": 10.328621847701617,
+      "learning_rate": 2.1354166666666666e-05,
+      "loss": 3.2333,
+      "step": 42
+    },
+    {
+      "epoch": 0.134375,
+      "grad_norm": 16.479079765607736,
+      "learning_rate": 2.1875e-05,
+      "loss": 3.0211,
+      "step": 43
+    },
+    {
+      "epoch": 0.1375,
+      "grad_norm": 8.92052911030536,
+      "learning_rate": 2.2395833333333337e-05,
+      "loss": 2.8675,
+      "step": 44
+    },
+    {
+      "epoch": 0.140625,
+      "grad_norm": 9.413102969331767,
+      "learning_rate": 2.2916666666666667e-05,
+      "loss": 2.9005,
+      "step": 45
+    },
+    {
+      "epoch": 0.14375,
+      "grad_norm": 6.115916371403105,
+      "learning_rate": 2.34375e-05,
+      "loss": 2.6911,
+      "step": 46
+    },
+    {
+      "epoch": 0.146875,
+      "grad_norm": 13.816870107125734,
+      "learning_rate": 2.3958333333333334e-05,
+      "loss": 2.7078,
+      "step": 47
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 7.562557833546146,
+      "learning_rate": 2.4479166666666668e-05,
+      "loss": 2.6333,
+      "step": 48
+    },
+    {
+      "epoch": 0.153125,
+      "grad_norm": 12.713068145664923,
+      "learning_rate": 2.5e-05,
+      "loss": 2.748,
+      "step": 49
+    },
+    {
+      "epoch": 0.15625,
+      "grad_norm": 12.926801333868049,
+      "learning_rate": 2.552083333333333e-05,
+      "loss": 2.5928,
+      "step": 50
+    },
+    {
+      "epoch": 0.159375,
+      "grad_norm": 15.342784916259383,
+      "learning_rate": 2.604166666666667e-05,
+      "loss": 2.7113,
+      "step": 51
+    },
+    {
+      "epoch": 0.1625,
+      "grad_norm": 11.90439963783374,
+      "learning_rate": 2.6562500000000002e-05,
+      "loss": 2.5776,
+      "step": 52
+    },
+    {
+      "epoch": 0.165625,
+      "grad_norm": 9.381560076123646,
+      "learning_rate": 2.7083333333333332e-05,
+      "loss": 2.4661,
+      "step": 53
+    },
+    {
+      "epoch": 0.16875,
+      "grad_norm": 14.841223460677213,
+      "learning_rate": 2.760416666666667e-05,
+      "loss": 2.3863,
+      "step": 54
+    },
+    {
+      "epoch": 0.171875,
+      "grad_norm": 9.76874980773837,
+      "learning_rate": 2.8125000000000003e-05,
+      "loss": 2.2455,
+      "step": 55
+    },
+    {
+      "epoch": 0.175,
+      "grad_norm": 13.147569869091708,
+      "learning_rate": 2.8645833333333333e-05,
+      "loss": 2.2638,
+      "step": 56
+    },
+    {
+      "epoch": 0.178125,
+      "grad_norm": 10.469007478544903,
+      "learning_rate": 2.916666666666667e-05,
+      "loss": 2.1291,
+      "step": 57
+    },
+    {
+      "epoch": 0.18125,
+      "grad_norm": 13.373774314984846,
+      "learning_rate": 2.96875e-05,
+      "loss": 2.2404,
+      "step": 58
+    },
+    {
+      "epoch": 0.184375,
+      "grad_norm": 12.113179340675496,
+      "learning_rate": 3.0208333333333334e-05,
+      "loss": 2.065,
+      "step": 59
+    },
+    {
+      "epoch": 0.1875,
+      "grad_norm": 10.485190352155438,
+      "learning_rate": 3.072916666666667e-05,
+      "loss": 2.0951,
+      "step": 60
+    },
+    {
+      "epoch": 0.190625,
+      "grad_norm": 8.341343325946616,
+      "learning_rate": 3.125e-05,
+      "loss": 2.022,
+      "step": 61
+    },
+    {
+      "epoch": 0.19375,
+      "grad_norm": 12.387319066452953,
+      "learning_rate": 3.177083333333333e-05,
+      "loss": 2.0098,
+      "step": 62
+    },
+    {
+      "epoch": 0.196875,
+      "grad_norm": 10.540581708049434,
+      "learning_rate": 3.229166666666667e-05,
+      "loss": 1.9363,
+      "step": 63
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 10.812810655996522,
+      "learning_rate": 3.2812500000000005e-05,
+      "loss": 1.8725,
+      "step": 64
+    },
+    {
+      "epoch": 0.203125,
+      "grad_norm": 9.484416715223766,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.9178,
+      "step": 65
+    },
+    {
+      "epoch": 0.20625,
+      "grad_norm": 10.790181395136832,
+      "learning_rate": 3.385416666666667e-05,
+      "loss": 1.8782,
+      "step": 66
+    },
+    {
+      "epoch": 0.209375,
+      "grad_norm": 10.235575961475147,
+      "learning_rate": 3.4375e-05,
+      "loss": 1.876,
+      "step": 67
+    },
+    {
+      "epoch": 0.2125,
+      "grad_norm": 6.271805190921046,
+      "learning_rate": 3.489583333333333e-05,
+      "loss": 1.7577,
+      "step": 68
+    },
+    {
+      "epoch": 0.215625,
+      "grad_norm": 9.316287124115359,
+      "learning_rate": 3.541666666666667e-05,
+      "loss": 1.7794,
+      "step": 69
+    },
+    {
+      "epoch": 0.21875,
+      "grad_norm": 6.281571641162089,
+      "learning_rate": 3.59375e-05,
+      "loss": 1.7455,
+      "step": 70
+    },
+    {
+      "epoch": 0.221875,
+      "grad_norm": 9.05622087806547,
+      "learning_rate": 3.6458333333333336e-05,
+      "loss": 1.8924,
+      "step": 71
+    },
+    {
+      "epoch": 0.225,
+      "grad_norm": 5.776852355314147,
+      "learning_rate": 3.697916666666667e-05,
+      "loss": 1.7291,
+      "step": 72
+    },
+    {
+      "epoch": 0.228125,
+      "grad_norm": 11.682603554859547,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 1.7906,
+      "step": 73
+    },
+    {
+      "epoch": 0.23125,
+      "grad_norm": 12.163996053395278,
+      "learning_rate": 3.8020833333333334e-05,
+      "loss": 1.7172,
+      "step": 74
+    },
+    {
+      "epoch": 0.234375,
+      "grad_norm": 6.560030873936021,
+      "learning_rate": 3.854166666666667e-05,
+      "loss": 1.617,
+      "step": 75
+    },
+    {
+      "epoch": 0.2375,
+      "grad_norm": 14.58298052284849,
+      "learning_rate": 3.90625e-05,
+      "loss": 1.6862,
+      "step": 76
+    },
+    {
+      "epoch": 0.240625,
+      "grad_norm": 9.78098581901162,
+      "learning_rate": 3.958333333333333e-05,
+      "loss": 1.671,
+      "step": 77
+    },
+    {
+      "epoch": 0.24375,
+      "grad_norm": 10.706052734180874,
+      "learning_rate": 4.010416666666667e-05,
+      "loss": 1.6405,
+      "step": 78
+    },
+    {
+      "epoch": 0.246875,
+      "grad_norm": 11.379102278294289,
+      "learning_rate": 4.0625000000000005e-05,
+      "loss": 1.6451,
+      "step": 79
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 7.221638254380996,
+      "learning_rate": 4.1145833333333335e-05,
+      "loss": 1.6891,
+      "step": 80
+    },
+    {
+      "epoch": 0.253125,
+      "grad_norm": 22.808201903647433,
+      "learning_rate": 4.166666666666667e-05,
+      "loss": 1.7122,
+      "step": 81
+    },
+    {
+      "epoch": 0.25625,
+      "grad_norm": 18.639216075641873,
+      "learning_rate": 4.21875e-05,
+      "loss": 1.8257,
+      "step": 82
+    },
+    {
+      "epoch": 0.259375,
+      "grad_norm": 10.72731520702505,
+      "learning_rate": 4.270833333333333e-05,
+      "loss": 1.5362,
+      "step": 83
+    },
+    {
+      "epoch": 0.2625,
+      "grad_norm": 15.545323971414044,
+      "learning_rate": 4.322916666666667e-05,
+      "loss": 1.6213,
+      "step": 84
+    },
+    {
+      "epoch": 0.265625,
+      "grad_norm": 11.065955184771042,
+      "learning_rate": 4.375e-05,
+      "loss": 1.5553,
+      "step": 85
+    },
+    {
+      "epoch": 0.26875,
+      "grad_norm": 11.20654869730552,
+      "learning_rate": 4.4270833333333337e-05,
+      "loss": 1.5246,
+      "step": 86
+    },
+    {
+      "epoch": 0.271875,
+      "grad_norm": 11.932483781648322,
+      "learning_rate": 4.4791666666666673e-05,
+      "loss": 1.5369,
+      "step": 87
+    },
+    {
+      "epoch": 0.275,
+      "grad_norm": 11.752405651566585,
+      "learning_rate": 4.5312500000000004e-05,
+      "loss": 1.6348,
+      "step": 88
+    },
+    {
+      "epoch": 0.278125,
+      "grad_norm": 11.711495480244865,
+      "learning_rate": 4.5833333333333334e-05,
+      "loss": 1.5125,
+      "step": 89
+    },
+    {
+      "epoch": 0.28125,
+      "grad_norm": 8.922074770695046,
+      "learning_rate": 4.635416666666667e-05,
+      "loss": 1.4361,
+      "step": 90
+    },
+    {
+      "epoch": 0.284375,
+      "grad_norm": 10.220098410430548,
+      "learning_rate": 4.6875e-05,
+      "loss": 1.4668,
+      "step": 91
+    },
+    {
+      "epoch": 0.2875,
+      "grad_norm": 8.746695342588895,
+      "learning_rate": 4.739583333333333e-05,
+      "loss": 1.4902,
+      "step": 92
+    },
+    {
+      "epoch": 0.290625,
+      "grad_norm": 12.109046065683124,
+      "learning_rate": 4.791666666666667e-05,
+      "loss": 1.3935,
+      "step": 93
+    },
+    {
+      "epoch": 0.29375,
+      "grad_norm": 10.592432360976927,
+      "learning_rate": 4.8437500000000005e-05,
+      "loss": 1.4936,
+      "step": 94
+    },
+    {
+      "epoch": 0.296875,
+      "grad_norm": 13.401347502550934,
+      "learning_rate": 4.8958333333333335e-05,
+      "loss": 1.4997,
+      "step": 95
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 13.708865595809112,
+      "learning_rate": 4.947916666666667e-05,
+      "loss": 1.4163,
+      "step": 96
+    },
+    {
+      "epoch": 0.303125,
+      "grad_norm": 8.119460814837431,
+      "learning_rate": 5e-05,
+      "loss": 1.3589,
+      "step": 97
+    },
+    {
+      "epoch": 0.30625,
+      "grad_norm": 16.933039368228076,
+      "learning_rate": 5.052083333333334e-05,
+      "loss": 1.4299,
+      "step": 98
+    },
+    {
+      "epoch": 0.309375,
+      "grad_norm": 13.150613444325701,
+      "learning_rate": 5.104166666666666e-05,
+      "loss": 1.3481,
+      "step": 99
+    },
+    {
+      "epoch": 0.3125,
+      "grad_norm": 8.912850450078942,
+      "learning_rate": 5.15625e-05,
+      "loss": 1.362,
+      "step": 100
+    },
+    {
+      "epoch": 0.315625,
+      "grad_norm": 13.127269697799717,
+      "learning_rate": 5.208333333333334e-05,
+      "loss": 1.2415,
+      "step": 101
+    },
+    {
+      "epoch": 0.31875,
+      "grad_norm": 6.1349553392940726,
+      "learning_rate": 5.260416666666667e-05,
+      "loss": 1.2637,
+      "step": 102
+    },
+    {
+      "epoch": 0.321875,
+      "grad_norm": 23.625786709924544,
+      "learning_rate": 5.3125000000000004e-05,
+      "loss": 1.6585,
+      "step": 103
+    },
+    {
+      "epoch": 0.325,
+      "grad_norm": 24.150218381183,
+      "learning_rate": 5.364583333333334e-05,
+      "loss": 1.4994,
+      "step": 104
+    },
+    {
+      "epoch": 0.328125,
+      "grad_norm": 9.573494256152,
+      "learning_rate": 5.4166666666666664e-05,
+      "loss": 1.2312,
+      "step": 105
+    },
+    {
+      "epoch": 0.33125,
+      "grad_norm": 18.32248401936274,
+      "learning_rate": 5.46875e-05,
+      "loss": 1.4261,
+      "step": 106
+    },
+    {
+      "epoch": 0.334375,
+      "grad_norm": 22.759858471699914,
+      "learning_rate": 5.520833333333334e-05,
+      "loss": 1.5693,
+      "step": 107
+    },
+    {
+      "epoch": 0.3375,
+      "grad_norm": 14.325787209593175,
+      "learning_rate": 5.572916666666667e-05,
+      "loss": 1.3204,
+      "step": 108
+    },
+    {
+      "epoch": 0.340625,
+      "grad_norm": 9.161111873027071,
+      "learning_rate": 5.6250000000000005e-05,
+      "loss": 1.2344,
+      "step": 109
+    },
+    {
+      "epoch": 0.34375,
+      "grad_norm": 16.54378026170953,
+      "learning_rate": 5.677083333333334e-05,
+      "loss": 1.3876,
+      "step": 110
+    },
+    {
+      "epoch": 0.346875,
+      "grad_norm": 14.660224214393118,
+      "learning_rate": 5.7291666666666666e-05,
+      "loss": 1.3828,
+      "step": 111
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 4.746224404487515,
+      "learning_rate": 5.78125e-05,
+      "loss": 1.2078,
+      "step": 112
+    },
+    {
+      "epoch": 0.353125,
+      "grad_norm": 10.083095364787159,
+      "learning_rate": 5.833333333333334e-05,
+      "loss": 1.252,
+      "step": 113
+    },
+    {
+      "epoch": 0.35625,
+      "grad_norm": 3.744430925274016,
+      "learning_rate": 5.885416666666666e-05,
+      "loss": 1.1536,
+      "step": 114
+    },
+    {
+      "epoch": 0.359375,
+      "grad_norm": 7.1875275730879125,
+      "learning_rate": 5.9375e-05,
+      "loss": 1.1371,
+      "step": 115
+    },
+    {
+      "epoch": 0.3625,
+      "grad_norm": 6.558006448835113,
+      "learning_rate": 5.989583333333334e-05,
+      "loss": 1.13,
+      "step": 116
+    },
+    {
+      "epoch": 0.365625,
+      "grad_norm": 9.733359886218526,
+      "learning_rate": 6.041666666666667e-05,
+      "loss": 1.1528,
+      "step": 117
+    },
+    {
+      "epoch": 0.36875,
+      "grad_norm": 9.76291297898059,
+      "learning_rate": 6.0937500000000004e-05,
+      "loss": 1.0878,
+      "step": 118
+    },
+    {
+      "epoch": 0.371875,
+      "grad_norm": 6.2437666568877805,
+      "learning_rate": 6.145833333333334e-05,
+      "loss": 1.0861,
+      "step": 119
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 8.199907557931391,
+      "learning_rate": 6.197916666666667e-05,
+      "loss": 1.0335,
+      "step": 120
+    },
+    {
+      "epoch": 0.378125,
+      "grad_norm": 10.499996193190483,
+      "learning_rate": 6.25e-05,
+      "loss": 1.1987,
+      "step": 121
+    },
+    {
+      "epoch": 0.38125,
+      "grad_norm": 9.114347736660998,
+      "learning_rate": 6.302083333333335e-05,
+      "loss": 1.0914,
+      "step": 122
+    },
+    {
+      "epoch": 0.384375,
+      "grad_norm": 8.268829599928603,
+      "learning_rate": 6.354166666666666e-05,
+      "loss": 1.055,
+      "step": 123
+    },
+    {
+      "epoch": 0.3875,
+      "grad_norm": 10.125304154569957,
+      "learning_rate": 6.40625e-05,
+      "loss": 1.0882,
+      "step": 124
+    },
+    {
+      "epoch": 0.390625,
+      "grad_norm": 8.1220344052888,
+      "learning_rate": 6.458333333333334e-05,
+      "loss": 1.0011,
+      "step": 125
+    },
+    {
+      "epoch": 0.39375,
+      "grad_norm": 12.531667895600402,
+      "learning_rate": 6.510416666666667e-05,
+      "loss": 1.1001,
+      "step": 126
+    },
+    {
+      "epoch": 0.396875,
+      "grad_norm": 7.001301118095523,
+      "learning_rate": 6.562500000000001e-05,
+      "loss": 1.0586,
+      "step": 127
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 13.131918744229115,
+      "learning_rate": 6.614583333333334e-05,
+      "loss": 1.1934,
+      "step": 128
+    },
+    {
+      "epoch": 0.403125,
+      "grad_norm": 8.655929689449874,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 0.9972,
+      "step": 129
+    },
+    {
+      "epoch": 0.40625,
+      "grad_norm": 16.011683259014738,
+      "learning_rate": 6.71875e-05,
+      "loss": 1.1322,
+      "step": 130
+    },
+    {
+      "epoch": 0.409375,
+      "grad_norm": 13.459517614378198,
+      "learning_rate": 6.770833333333334e-05,
+      "loss": 1.0996,
+      "step": 131
+    },
+    {
+      "epoch": 0.4125,
+      "grad_norm": 8.184600225776215,
+      "learning_rate": 6.822916666666666e-05,
+      "loss": 0.9241,
+      "step": 132
+    },
+    {
+      "epoch": 0.415625,
+      "grad_norm": 9.770142187366993,
+      "learning_rate": 6.875e-05,
+      "loss": 0.9989,
+      "step": 133
+    },
+    {
+      "epoch": 0.41875,
+      "grad_norm": 8.680458648025033,
+      "learning_rate": 6.927083333333333e-05,
+      "loss": 1.0223,
+      "step": 134
+    },
+    {
+      "epoch": 0.421875,
+      "grad_norm": 8.262315759784606,
+      "learning_rate": 6.979166666666666e-05,
+      "loss": 1.0372,
+      "step": 135
+    },
+    {
+      "epoch": 0.425,
+      "grad_norm": 8.229942662737823,
+      "learning_rate": 7.031250000000001e-05,
+      "loss": 0.9867,
+      "step": 136
+    },
+    {
+      "epoch": 0.428125,
+      "grad_norm": 7.2390333730582,
+      "learning_rate": 7.083333333333334e-05,
+      "loss": 1.0357,
+      "step": 137
+    },
+    {
+      "epoch": 0.43125,
+      "grad_norm": 7.925478703298735,
+      "learning_rate": 7.135416666666667e-05,
+      "loss": 0.9554,
+      "step": 138
+    },
+    {
+      "epoch": 0.434375,
+      "grad_norm": 5.948018489055187,
+      "learning_rate": 7.1875e-05,
+      "loss": 0.9297,
+      "step": 139
+    },
+    {
+      "epoch": 0.4375,
+      "grad_norm": 7.868847102053355,
+      "learning_rate": 7.239583333333334e-05,
+      "loss": 0.9197,
+      "step": 140
+    },
+    {
+      "epoch": 0.440625,
+      "grad_norm": 4.0931309918679855,
+      "learning_rate": 7.291666666666667e-05,
+      "loss": 0.8842,
+      "step": 141
+    },
+    {
+      "epoch": 0.44375,
+      "grad_norm": 11.977179896611627,
+      "learning_rate": 7.34375e-05,
+      "loss": 0.9692,
+      "step": 142
+    },
+    {
+      "epoch": 0.446875,
+      "grad_norm": 6.962877179309973,
+      "learning_rate": 7.395833333333335e-05,
+      "loss": 0.9198,
+      "step": 143
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 11.404784564808287,
+      "learning_rate": 7.447916666666666e-05,
+      "loss": 0.9714,
+      "step": 144
+    },
+    {
+      "epoch": 0.453125,
+      "grad_norm": 9.53013227271108,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.9551,
+      "step": 145
+    },
+    {
+      "epoch": 0.45625,
+      "grad_norm": 7.852769865014313,
+      "learning_rate": 7.552083333333334e-05,
+      "loss": 0.9476,
+      "step": 146
+    },
+    {
+      "epoch": 0.459375,
+      "grad_norm": 12.027549904258692,
+      "learning_rate": 7.604166666666667e-05,
+      "loss": 0.9747,
+      "step": 147
+    },
+    {
+      "epoch": 0.4625,
+      "grad_norm": 8.4280228515814,
+      "learning_rate": 7.65625e-05,
+      "loss": 0.9212,
+      "step": 148
+    },
+    {
+      "epoch": 0.465625,
+      "grad_norm": 5.36717019992068,
+      "learning_rate": 7.708333333333334e-05,
+      "loss": 0.8841,
+      "step": 149
+    },
+    {
+      "epoch": 0.46875,
+      "grad_norm": 9.707836798671655,
+      "learning_rate": 7.760416666666667e-05,
+      "loss": 0.9445,
+      "step": 150
+    },
+    {
+      "epoch": 0.471875,
+      "grad_norm": 7.8211950493386375,
+      "learning_rate": 7.8125e-05,
+      "loss": 0.9346,
+      "step": 151
+    },
+    {
+      "epoch": 0.475,
+      "grad_norm": 10.00239033844591,
+      "learning_rate": 7.864583333333335e-05,
+      "loss": 0.9327,
+      "step": 152
+    },
+    {
+      "epoch": 0.478125,
+      "grad_norm": 7.080184630683423,
+      "learning_rate": 7.916666666666666e-05,
+      "loss": 0.8713,
+      "step": 153
+    },
+    {
+      "epoch": 0.48125,
+      "grad_norm": 10.959277685948559,
+      "learning_rate": 7.96875e-05,
+      "loss": 0.9879,
+      "step": 154
+    },
+    {
+      "epoch": 0.484375,
+      "grad_norm": 8.406304115385007,
+      "learning_rate": 8.020833333333334e-05,
+      "loss": 0.9298,
+      "step": 155
+    },
+    {
+      "epoch": 0.4875,
+      "grad_norm": 10.857922003791755,
+      "learning_rate": 8.072916666666667e-05,
+      "loss": 0.8592,
+      "step": 156
+    },
+    {
+      "epoch": 0.490625,
+      "grad_norm": 10.507698569336473,
+      "learning_rate": 8.125000000000001e-05,
+      "loss": 0.9309,
+      "step": 157
+    },
+    {
+      "epoch": 0.49375,
+      "grad_norm": 6.529540954550196,
+      "learning_rate": 8.177083333333334e-05,
+      "loss": 0.8305,
+      "step": 158
+    },
+    {
+      "epoch": 0.496875,
+      "grad_norm": 8.352235620067676,
+      "learning_rate": 8.229166666666667e-05,
+      "loss": 0.9641,
+      "step": 159
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 4.657798765262621,
+      "learning_rate": 8.28125e-05,
+      "loss": 0.8663,
+      "step": 160
+    },
+    {
+      "epoch": 0.503125,
+      "grad_norm": 6.6468076014241335,
+      "learning_rate": 8.333333333333334e-05,
+      "loss": 0.8804,
+      "step": 161
+    },
+    {
+      "epoch": 0.50625,
+      "grad_norm": 8.18281474088343,
+      "learning_rate": 8.385416666666666e-05,
+      "loss": 0.9127,
+      "step": 162
+    },
+    {
+      "epoch": 0.509375,
+      "grad_norm": 5.149736076594657,
+      "learning_rate": 8.4375e-05,
+      "loss": 0.9288,
+      "step": 163
+    },
+    {
+      "epoch": 0.5125,
+      "grad_norm": 7.581979286907847,
+      "learning_rate": 8.489583333333333e-05,
+      "loss": 0.8483,
+      "step": 164
+    },
+    {
+      "epoch": 0.515625,
+      "grad_norm": 5.669787431334474,
+      "learning_rate": 8.541666666666666e-05,
+      "loss": 0.7873,
+      "step": 165
+    },
+    {
+      "epoch": 0.51875,
+      "grad_norm": 9.260330708502003,
+      "learning_rate": 8.593750000000001e-05,
+      "loss": 0.9492,
+      "step": 166
+    },
+    {
+      "epoch": 0.521875,
+      "grad_norm": 5.789639924996383,
+      "learning_rate": 8.645833333333334e-05,
+      "loss": 0.8318,
+      "step": 167
+    },
+    {
+      "epoch": 0.525,
+      "grad_norm": 10.229343902621522,
+      "learning_rate": 8.697916666666667e-05,
+      "loss": 0.9097,
+      "step": 168
+    },
+    {
+      "epoch": 0.528125,
+      "grad_norm": 9.228206884119015,
+      "learning_rate": 8.75e-05,
+      "loss": 0.8582,
+      "step": 169
+    },
+    {
+      "epoch": 0.53125,
+      "grad_norm": 7.666777700421669,
+      "learning_rate": 8.802083333333334e-05,
+      "loss": 0.8492,
+      "step": 170
+    },
+    {
+      "epoch": 0.534375,
+      "grad_norm": 7.498065591632684,
+      "learning_rate": 8.854166666666667e-05,
+      "loss": 0.8115,
+      "step": 171
+    },
+    {
+      "epoch": 0.5375,
+      "grad_norm": 6.916558374382635,
+      "learning_rate": 8.90625e-05,
+      "loss": 0.7908,
+      "step": 172
+    },
+    {
+      "epoch": 0.540625,
+      "grad_norm": 7.915912152903943,
+      "learning_rate": 8.958333333333335e-05,
+      "loss": 0.8059,
+      "step": 173
+    },
+    {
+      "epoch": 0.54375,
+      "grad_norm": 5.9954176654563485,
+      "learning_rate": 9.010416666666666e-05,
+      "loss": 0.8024,
+      "step": 174
+    },
+    {
+      "epoch": 0.546875,
+      "grad_norm": 5.173849203895654,
+      "learning_rate": 9.062500000000001e-05,
+      "loss": 0.7418,
+      "step": 175
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 4.146423153339542,
+      "learning_rate": 9.114583333333334e-05,
+      "loss": 0.7816,
+      "step": 176
+    },
+    {
+      "epoch": 0.553125,
+      "grad_norm": 10.445546819978581,
+      "learning_rate": 9.166666666666667e-05,
+      "loss": 0.8374,
+      "step": 177
+    },
+    {
+      "epoch": 0.55625,
+      "grad_norm": 5.977128755691389,
+      "learning_rate": 9.21875e-05,
+      "loss": 0.7962,
+      "step": 178
+    },
+    {
+      "epoch": 0.559375,
+      "grad_norm": 9.020297195137891,
+      "learning_rate": 9.270833333333334e-05,
+      "loss": 0.865,
+      "step": 179
+    },
+    {
+      "epoch": 0.5625,
+      "grad_norm": 7.106875300447416,
+      "learning_rate": 9.322916666666667e-05,
+      "loss": 0.7876,
+      "step": 180
+    },
+    {
+      "epoch": 0.565625,
+      "grad_norm": 9.329673526456457,
+      "learning_rate": 9.375e-05,
+      "loss": 0.8542,
+      "step": 181
+    },
+    {
+      "epoch": 0.56875,
+      "grad_norm": 6.58668399217218,
+      "learning_rate": 9.427083333333335e-05,
+      "loss": 0.8382,
+      "step": 182
+    },
+    {
+      "epoch": 0.571875,
+      "grad_norm": 10.083483331258801,
+      "learning_rate": 9.479166666666666e-05,
+      "loss": 0.875,
+      "step": 183
+    },
+    {
+      "epoch": 0.575,
+      "grad_norm": 5.169975483021176,
+      "learning_rate": 9.53125e-05,
+      "loss": 0.8075,
+      "step": 184
+    },
+    {
+      "epoch": 0.578125,
+      "grad_norm": 10.925234205175018,
+      "learning_rate": 9.583333333333334e-05,
+      "loss": 0.7896,
+      "step": 185
+    },
+    {
+      "epoch": 0.58125,
+      "grad_norm": 9.453839262647373,
+      "learning_rate": 9.635416666666667e-05,
+      "loss": 0.8686,
+      "step": 186
+    },
+    {
+      "epoch": 0.584375,
+      "grad_norm": 7.51176543895577,
+      "learning_rate": 9.687500000000001e-05,
+      "loss": 0.7349,
+      "step": 187
+    },
+    {
+      "epoch": 0.5875,
+      "grad_norm": 8.371249075947729,
+      "learning_rate": 9.739583333333334e-05,
+      "loss": 0.7885,
+      "step": 188
+    },
+    {
+      "epoch": 0.590625,
+      "grad_norm": 6.2623315034271005,
+      "learning_rate": 9.791666666666667e-05,
+      "loss": 0.8232,
+      "step": 189
+    },
+    {
+      "epoch": 0.59375,
+      "grad_norm": 8.51607024992856,
+      "learning_rate": 9.84375e-05,
+      "loss": 0.7814,
+      "step": 190
+    },
+    {
+      "epoch": 0.596875,
+      "grad_norm": 5.969121833596023,
+      "learning_rate": 9.895833333333334e-05,
+      "loss": 0.7853,
+      "step": 191
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 6.931397740654763,
+      "learning_rate": 9.947916666666666e-05,
+      "loss": 0.8034,
+      "step": 192
+    },
+    {
+      "epoch": 0.603125,
+      "grad_norm": 5.8365666787524,
+      "learning_rate": 0.0001,
+      "loss": 0.7447,
+      "step": 193
+    },
+    {
+      "epoch": 0.60625,
+      "grad_norm": 6.4445007992856524,
+      "learning_rate": 0.00010052083333333333,
+      "loss": 0.7553,
+      "step": 194
+    },
+    {
+      "epoch": 0.609375,
+      "grad_norm": 7.808743283800006,
+      "learning_rate": 0.00010104166666666668,
+      "loss": 0.739,
+      "step": 195
+    },
+    {
+      "epoch": 0.6125,
+      "grad_norm": 5.226152562462588,
+      "learning_rate": 0.00010156250000000001,
+      "loss": 0.7505,
+      "step": 196
+    },
+    {
+      "epoch": 0.615625,
+      "grad_norm": 5.871676266320986,
+      "learning_rate": 0.00010208333333333333,
+      "loss": 0.7383,
+      "step": 197
+    },
+    {
+      "epoch": 0.61875,
+      "grad_norm": 7.690100226914687,
+      "learning_rate": 0.00010260416666666668,
+      "loss": 0.8546,
+      "step": 198
+    },
+    {
+      "epoch": 0.621875,
+      "grad_norm": 6.093202895282048,
+      "learning_rate": 0.000103125,
+      "loss": 0.7176,
+      "step": 199
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 6.227266062421436,
+      "learning_rate": 0.00010364583333333333,
+      "loss": 0.7289,
+      "step": 200
+    },
+    {
+      "epoch": 0.628125,
+      "grad_norm": 4.0853238976344315,
+      "learning_rate": 0.00010416666666666667,
+      "loss": 0.7418,
+      "step": 201
+    },
+    {
+      "epoch": 0.63125,
+      "grad_norm": 5.401743145574044,
+      "learning_rate": 0.0001046875,
+      "loss": 0.7138,
+      "step": 202
+    },
+    {
+      "epoch": 0.634375,
+      "grad_norm": 5.732632372228023,
+      "learning_rate": 0.00010520833333333333,
+      "loss": 0.696,
+      "step": 203
+    },
+    {
+      "epoch": 0.6375,
+      "grad_norm": 4.470471973395256,
+      "learning_rate": 0.00010572916666666668,
+      "loss": 0.747,
+      "step": 204
+    },
+    {
+      "epoch": 0.640625,
+      "grad_norm": 6.0144031305556345,
+      "learning_rate": 0.00010625000000000001,
+      "loss": 0.7014,
+      "step": 205
+    },
+    {
+      "epoch": 0.64375,
+      "grad_norm": 4.712532304774911,
+      "learning_rate": 0.00010677083333333334,
+      "loss": 0.676,
+      "step": 206
+    },
+    {
+      "epoch": 0.646875,
+      "grad_norm": 6.243704020795876,
+      "learning_rate": 0.00010729166666666668,
+      "loss": 0.6455,
+      "step": 207
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 6.1633651675976076,
+      "learning_rate": 0.00010781250000000001,
+      "loss": 0.7559,
+      "step": 208
+    },
+    {
+      "epoch": 0.653125,
+      "grad_norm": 6.437422951687512,
+      "learning_rate": 0.00010833333333333333,
+      "loss": 0.6652,
+      "step": 209
+    },
+    {
+      "epoch": 0.65625,
+      "grad_norm": 6.060227515981516,
+      "learning_rate": 0.00010885416666666669,
+      "loss": 0.7186,
+      "step": 210
+    },
+    {
+      "epoch": 0.659375,
+      "grad_norm": 4.347124507702208,
+      "learning_rate": 0.000109375,
+      "loss": 0.7089,
+      "step": 211
+    },
+    {
+      "epoch": 0.6625,
+      "grad_norm": 9.86919359385419,
+      "learning_rate": 0.00010989583333333333,
+      "loss": 0.7651,
+      "step": 212
+    },
+    {
+      "epoch": 0.665625,
+      "grad_norm": 8.099230221322795,
+      "learning_rate": 0.00011041666666666668,
+      "loss": 0.7662,
+      "step": 213
+    },
+    {
+      "epoch": 0.66875,
+      "grad_norm": 9.982836658924466,
+      "learning_rate": 0.0001109375,
+      "loss": 0.7726,
+      "step": 214
+    },
+    {
+      "epoch": 0.671875,
+      "grad_norm": 9.11377532717914,
+      "learning_rate": 0.00011145833333333334,
+      "loss": 0.732,
+      "step": 215
+    },
+    {
+      "epoch": 0.675,
+      "grad_norm": 7.7224363657361135,
+      "learning_rate": 0.00011197916666666668,
+      "loss": 0.7448,
+      "step": 216
+    },
+    {
+      "epoch": 0.678125,
+      "grad_norm": 7.264877304875326,
+      "learning_rate": 0.00011250000000000001,
+      "loss": 0.7154,
+      "step": 217
+    },
+    {
+      "epoch": 0.68125,
+      "grad_norm": 5.968019429647152,
+      "learning_rate": 0.00011302083333333333,
+      "loss": 0.6751,
+      "step": 218
+    },
+    {
+      "epoch": 0.684375,
+      "grad_norm": 5.326970462516753,
+      "learning_rate": 0.00011354166666666668,
+      "loss": 0.7143,
+      "step": 219
+    },
+    {
+      "epoch": 0.6875,
+      "grad_norm": 8.915861068302076,
+      "learning_rate": 0.0001140625,
+      "loss": 0.7451,
+      "step": 220
+    },
+    {
+      "epoch": 0.690625,
+      "grad_norm": 6.7277062630103925,
+      "learning_rate": 0.00011458333333333333,
+      "loss": 0.7089,
+      "step": 221
+    },
+    {
+      "epoch": 0.69375,
+      "grad_norm": 7.774681670860603,
+      "learning_rate": 0.00011510416666666667,
+      "loss": 0.7742,
+      "step": 222
+    },
+    {
+      "epoch": 0.696875,
+      "grad_norm": 7.723415561234424,
+      "learning_rate": 0.000115625,
+      "loss": 0.7436,
+      "step": 223
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 4.408925309735434,
+      "learning_rate": 0.00011614583333333334,
+      "loss": 0.6783,
+      "step": 224
+    },
+    {
+      "epoch": 0.703125,
+      "grad_norm": 5.128006412826596,
+      "learning_rate": 0.00011666666666666668,
+      "loss": 0.6574,
+      "step": 225
+    },
+    {
+      "epoch": 0.70625,
+      "grad_norm": 7.109322132051973,
+      "learning_rate": 0.00011718750000000001,
+      "loss": 0.6809,
+      "step": 226
+    },
+    {
+      "epoch": 0.709375,
+      "grad_norm": 5.678260661672194,
+      "learning_rate": 0.00011770833333333333,
+      "loss": 0.7801,
+      "step": 227
+    },
+    {
+      "epoch": 0.7125,
+      "grad_norm": 5.230539784787929,
+      "learning_rate": 0.00011822916666666668,
+      "loss": 0.6834,
+      "step": 228
+    },
+    {
+      "epoch": 0.715625,
+      "grad_norm": 4.298606538054625,
+      "learning_rate": 0.00011875,
+      "loss": 0.6126,
+      "step": 229
+    },
+    {
+      "epoch": 0.71875,
+      "grad_norm": 5.823181840932161,
+      "learning_rate": 0.00011927083333333333,
+      "loss": 0.7571,
+      "step": 230
+    },
+    {
+      "epoch": 0.721875,
+      "grad_norm": 3.557763682325525,
+      "learning_rate": 0.00011979166666666667,
+      "loss": 0.6532,
+      "step": 231
+    },
+    {
+      "epoch": 0.725,
+      "grad_norm": 9.360009235239575,
+      "learning_rate": 0.0001203125,
+      "loss": 0.779,
+      "step": 232
+    },
+    {
+      "epoch": 0.728125,
+      "grad_norm": 6.467134528380297,
+      "learning_rate": 0.00012083333333333333,
+      "loss": 0.7027,
+      "step": 233
+    },
+    {
+      "epoch": 0.73125,
+      "grad_norm": 6.443330255081421,
+      "learning_rate": 0.00012135416666666668,
+      "loss": 0.7535,
+      "step": 234
+    },
+    {
+      "epoch": 0.734375,
+      "grad_norm": 5.263696964696843,
+      "learning_rate": 0.00012187500000000001,
+      "loss": 0.7076,
+      "step": 235
+    },
+    {
+      "epoch": 0.7375,
+      "grad_norm": 8.41674251103533,
+      "learning_rate": 0.00012239583333333332,
+      "loss": 0.7685,
+      "step": 236
+    },
+    {
+      "epoch": 0.740625,
+      "grad_norm": 6.468022344397508,
+      "learning_rate": 0.00012291666666666668,
+      "loss": 0.664,
+      "step": 237
+    },
+    {
+      "epoch": 0.74375,
+      "grad_norm": 4.667797510411515,
+      "learning_rate": 0.0001234375,
+      "loss": 0.723,
+      "step": 238
+    },
+    {
+      "epoch": 0.746875,
+      "grad_norm": 5.095530201731687,
+      "learning_rate": 0.00012395833333333334,
+      "loss": 0.6788,
+      "step": 239
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 5.024524480040258,
+      "learning_rate": 0.00012447916666666667,
+      "loss": 0.7031,
+      "step": 240
+    },
+    {
+      "epoch": 0.753125,
+      "grad_norm": 4.686387781862867,
+      "learning_rate": 0.000125,
+      "loss": 0.6393,
+      "step": 241
+    },
+    {
+      "epoch": 0.75625,
+      "grad_norm": 4.147768654688554,
+      "learning_rate": 0.00012552083333333333,
+      "loss": 0.6721,
+      "step": 242
+    },
+    {
+      "epoch": 0.759375,
+      "grad_norm": 3.991069562808937,
+      "learning_rate": 0.0001260416666666667,
+      "loss": 0.5938,
+      "step": 243
+    },
+    {
+      "epoch": 0.7625,
+      "grad_norm": 8.149179025879748,
+      "learning_rate": 0.0001265625,
+      "loss": 0.6779,
+      "step": 244
+    },
+    {
+      "epoch": 0.765625,
+      "grad_norm": 5.3989756217933165,
+      "learning_rate": 0.00012708333333333332,
+      "loss": 0.653,
+      "step": 245
+    },
+    {
+      "epoch": 0.76875,
+      "grad_norm": 7.382240102146617,
+      "learning_rate": 0.00012760416666666668,
+      "loss": 0.765,
+      "step": 246
+    },
+    {
+      "epoch": 0.771875,
+      "grad_norm": 6.286650490681898,
+      "learning_rate": 0.000128125,
+      "loss": 0.7326,
+      "step": 247
+    },
+    {
+      "epoch": 0.775,
+      "grad_norm": 4.633473634760908,
+      "learning_rate": 0.00012864583333333334,
+      "loss": 0.6258,
+      "step": 248
+    },
+    {
+      "epoch": 0.778125,
+      "grad_norm": 10.673427073287195,
+      "learning_rate": 0.00012916666666666667,
+      "loss": 0.7801,
+      "step": 249
+    },
+    {
+      "epoch": 0.78125,
+      "grad_norm": 5.223890554435825,
+      "learning_rate": 0.0001296875,
+      "loss": 0.6659,
+      "step": 250
+    },
+    {
+      "epoch": 0.784375,
+      "grad_norm": 11.277996957765167,
+      "learning_rate": 0.00013020833333333333,
+      "loss": 0.8506,
+      "step": 251
+    },
+    {
+      "epoch": 0.7875,
+      "grad_norm": 8.192236976895307,
+      "learning_rate": 0.0001307291666666667,
+      "loss": 0.8054,
+      "step": 252
+    },
+    {
+      "epoch": 0.790625,
+      "grad_norm": 7.564377201542728,
+      "learning_rate": 0.00013125000000000002,
+      "loss": 0.7872,
+      "step": 253
+    },
+    {
+      "epoch": 0.79375,
+      "grad_norm": 7.376895934218807,
+      "learning_rate": 0.00013177083333333332,
+      "loss": 0.8117,
+      "step": 254
+    },
+    {
+      "epoch": 0.796875,
+      "grad_norm": 6.4492649766013574,
+      "learning_rate": 0.00013229166666666668,
+      "loss": 0.6726,
+      "step": 255
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 7.056223615199466,
+      "learning_rate": 0.0001328125,
+      "loss": 0.7024,
+      "step": 256
+    },
+    {
+      "epoch": 0.803125,
+      "grad_norm": 6.188905904504976,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 0.7158,
+      "step": 257
+    },
+    {
+      "epoch": 0.80625,
+      "grad_norm": 5.81486514980825,
+      "learning_rate": 0.00013385416666666667,
+      "loss": 0.6696,
+      "step": 258
+    },
+    {
+      "epoch": 0.809375,
+      "grad_norm": 6.408786300617925,
+      "learning_rate": 0.000134375,
+      "loss": 0.7378,
+      "step": 259
+    },
+    {
+      "epoch": 0.8125,
+      "grad_norm": 6.033101933587246,
+      "learning_rate": 0.00013489583333333333,
+      "loss": 0.5996,
+      "step": 260
+    },
+    {
+      "epoch": 0.815625,
+      "grad_norm": 4.806690666893966,
+      "learning_rate": 0.0001354166666666667,
+      "loss": 0.6503,
+      "step": 261
+    },
+    {
+      "epoch": 0.81875,
+      "grad_norm": 6.394692964269329,
+      "learning_rate": 0.00013593750000000002,
+      "loss": 0.72,
+      "step": 262
+    },
+    {
+      "epoch": 0.821875,
+      "grad_norm": 3.867322783879365,
+      "learning_rate": 0.00013645833333333332,
+      "loss": 0.6163,
+      "step": 263
+    },
+    {
+      "epoch": 0.825,
+      "grad_norm": 5.5139377490245876,
+      "learning_rate": 0.00013697916666666668,
+      "loss": 0.6186,
+      "step": 264
+    },
+    {
+      "epoch": 0.828125,
+      "grad_norm": 5.09570226545013,
+      "learning_rate": 0.0001375,
+      "loss": 0.6894,
+      "step": 265
+    },
+    {
+      "epoch": 0.83125,
+      "grad_norm": 3.8389028514679064,
+      "learning_rate": 0.00013802083333333334,
+      "loss": 0.6451,
+      "step": 266
+    },
+    {
+      "epoch": 0.834375,
+      "grad_norm": 4.577383334065461,
+      "learning_rate": 0.00013854166666666667,
+      "loss": 0.6337,
+      "step": 267
+    },
+    {
+      "epoch": 0.8375,
+      "grad_norm": 4.00685539620012,
+      "learning_rate": 0.0001390625,
+      "loss": 0.5703,
+      "step": 268
+    },
+    {
+      "epoch": 0.840625,
+      "grad_norm": 6.012861829383814,
+      "learning_rate": 0.00013958333333333333,
+      "loss": 0.6799,
+      "step": 269
+    },
+    {
+      "epoch": 0.84375,
+      "grad_norm": 3.552926641113641,
+      "learning_rate": 0.00014010416666666669,
+      "loss": 0.6019,
+      "step": 270
+    },
+    {
+      "epoch": 0.846875,
+      "grad_norm": 6.640586308933678,
+      "learning_rate": 0.00014062500000000002,
+      "loss": 0.6676,
+      "step": 271
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 5.863103082822842,
+      "learning_rate": 0.00014114583333333332,
+      "loss": 0.6962,
+      "step": 272
+    },
+    {
+      "epoch": 0.853125,
+      "grad_norm": 7.212282647310525,
+      "learning_rate": 0.00014166666666666668,
+      "loss": 0.7539,
+      "step": 273
+    },
+    {
+      "epoch": 0.85625,
+      "grad_norm": 6.138497137370133,
+      "learning_rate": 0.0001421875,
+      "loss": 0.6381,
+      "step": 274
+    },
+    {
+      "epoch": 0.859375,
+      "grad_norm": 5.075880610971953,
+      "learning_rate": 0.00014270833333333334,
+      "loss": 0.6947,
+      "step": 275
+    },
+    {
+      "epoch": 0.8625,
+      "grad_norm": 6.181891500305302,
+      "learning_rate": 0.00014322916666666667,
+      "loss": 0.6381,
+      "step": 276
+    },
+    {
+      "epoch": 0.865625,
+      "grad_norm": 5.250962298084777,
+      "learning_rate": 0.00014375,
+      "loss": 0.6293,
+      "step": 277
+    },
+    {
+      "epoch": 0.86875,
+      "grad_norm": 4.6093606156741,
+      "learning_rate": 0.00014427083333333333,
+      "loss": 0.6517,
+      "step": 278
+    },
+    {
+      "epoch": 0.871875,
+      "grad_norm": 4.112889069368331,
+      "learning_rate": 0.00014479166666666669,
+      "loss": 0.5992,
+      "step": 279
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 4.669767052226125,
+      "learning_rate": 0.00014531250000000002,
+      "loss": 0.6211,
+      "step": 280
+    },
+    {
+      "epoch": 0.878125,
+      "grad_norm": 4.196082806091218,
+      "learning_rate": 0.00014583333333333335,
+      "loss": 0.6721,
+      "step": 281
+    },
+    {
+      "epoch": 0.88125,
+      "grad_norm": 6.550932267913235,
+      "learning_rate": 0.00014635416666666668,
+      "loss": 0.7132,
+      "step": 282
+    },
+    {
+      "epoch": 0.884375,
+      "grad_norm": 3.7057766450929948,
+      "learning_rate": 0.000146875,
+      "loss": 0.609,
+      "step": 283
+    },
+    {
+      "epoch": 0.8875,
+      "grad_norm": 6.988619919422886,
+      "learning_rate": 0.00014739583333333334,
+      "loss": 0.6502,
+      "step": 284
+    },
+    {
+      "epoch": 0.890625,
+      "grad_norm": 5.1031894425980955,
+      "learning_rate": 0.0001479166666666667,
+      "loss": 0.6444,
+      "step": 285
+    },
+    {
+      "epoch": 0.89375,
+      "grad_norm": 7.3129173002131544,
+      "learning_rate": 0.0001484375,
+      "loss": 0.6916,
+      "step": 286
+    },
+    {
+      "epoch": 0.896875,
+      "grad_norm": 6.589299028651061,
+      "learning_rate": 0.00014895833333333333,
+      "loss": 0.6928,
+      "step": 287
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 6.507881019786985,
+      "learning_rate": 0.00014947916666666668,
+      "loss": 0.7102,
+      "step": 288
+    },
+    {
+      "epoch": 0.903125,
+      "grad_norm": 5.397882675964744,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 0.6867,
+      "step": 289
+    },
+    {
+      "epoch": 0.90625,
+      "grad_norm": 6.187071771361283,
+      "learning_rate": 0.00015052083333333334,
+      "loss": 0.6304,
+      "step": 290
+    },
+    {
+      "epoch": 0.909375,
+      "grad_norm": 5.1473279658105024,
+      "learning_rate": 0.00015104166666666667,
+      "loss": 0.6669,
+      "step": 291
+    },
+    {
+      "epoch": 0.9125,
+      "grad_norm": 5.952602708905966,
+      "learning_rate": 0.0001515625,
+      "loss": 0.6658,
+      "step": 292
+    },
+    {
+      "epoch": 0.915625,
+      "grad_norm": 4.833693946412644,
+      "learning_rate": 0.00015208333333333333,
+      "loss": 0.5923,
+      "step": 293
+    },
+    {
+      "epoch": 0.91875,
+      "grad_norm": 5.708733854302764,
+      "learning_rate": 0.0001526041666666667,
+      "loss": 0.6776,
+      "step": 294
+    },
+    {
+      "epoch": 0.921875,
+      "grad_norm": 6.159895020795588,
+      "learning_rate": 0.000153125,
+      "loss": 0.7081,
+      "step": 295
+    },
+    {
+      "epoch": 0.925,
+      "grad_norm": 3.6420861623581664,
+      "learning_rate": 0.00015364583333333333,
+      "loss": 0.5722,
+      "step": 296
+    },
+    {
+      "epoch": 0.928125,
+      "grad_norm": 4.498361402240788,
+      "learning_rate": 0.00015416666666666668,
+      "loss": 0.6347,
+      "step": 297
+    },
+    {
+      "epoch": 0.93125,
+      "grad_norm": 5.5407071416791025,
+      "learning_rate": 0.0001546875,
+      "loss": 0.6328,
+      "step": 298
+    },
+    {
+      "epoch": 0.934375,
+      "grad_norm": 2.989522367565342,
+      "learning_rate": 0.00015520833333333334,
+      "loss": 0.604,
+      "step": 299
+    },
+    {
+      "epoch": 0.9375,
+      "grad_norm": 7.212786015409645,
+      "learning_rate": 0.00015572916666666667,
+      "loss": 0.6996,
+      "step": 300
+    },
+    {
+      "epoch": 0.940625,
+      "grad_norm": 6.475186960727527,
+      "learning_rate": 0.00015625,
+      "loss": 0.6937,
+      "step": 301
+    },
+    {
+      "epoch": 0.94375,
+      "grad_norm": 4.876884136220253,
+      "learning_rate": 0.00015677083333333333,
+      "loss": 0.6748,
+      "step": 302
+    },
+    {
+      "epoch": 0.946875,
+      "grad_norm": 4.891452146223525,
+      "learning_rate": 0.0001572916666666667,
+      "loss": 0.6437,
+      "step": 303
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 4.553582576759581,
+      "learning_rate": 0.00015781250000000002,
+      "loss": 0.657,
+      "step": 304
+    },
+    {
+      "epoch": 0.953125,
+      "grad_norm": 7.094328124766603,
+      "learning_rate": 0.00015833333333333332,
+      "loss": 0.6877,
+      "step": 305
+    },
+    {
+      "epoch": 0.95625,
+      "grad_norm": 4.191489560484197,
+      "learning_rate": 0.00015885416666666668,
+      "loss": 0.6163,
+      "step": 306
+    },
+    {
+      "epoch": 0.959375,
+      "grad_norm": 9.641050857560009,
+      "learning_rate": 0.000159375,
+      "loss": 0.7381,
+      "step": 307
+    },
+    {
+      "epoch": 0.9625,
+      "grad_norm": 7.010284309022098,
+      "learning_rate": 0.00015989583333333334,
+      "loss": 0.7669,
+      "step": 308
+    },
+    {
+      "epoch": 0.965625,
+      "grad_norm": 5.124065842157194,
+      "learning_rate": 0.00016041666666666667,
+      "loss": 0.6315,
+      "step": 309
+    },
+    {
+      "epoch": 0.96875,
+      "grad_norm": 5.124058430454074,
+      "learning_rate": 0.0001609375,
+      "loss": 0.6506,
+      "step": 310
+    },
+    {
+      "epoch": 0.971875,
+      "grad_norm": 5.3050104754360525,
+      "learning_rate": 0.00016145833333333333,
+      "loss": 0.7087,
+      "step": 311
+    },
+    {
+      "epoch": 0.975,
+      "grad_norm": 4.98614798038132,
+      "learning_rate": 0.0001619791666666667,
+      "loss": 0.6785,
+      "step": 312
+    },
+    {
+      "epoch": 0.978125,
+      "grad_norm": 4.639878304940204,
+      "learning_rate": 0.00016250000000000002,
+      "loss": 0.6477,
+      "step": 313
+    },
+    {
+      "epoch": 0.98125,
+      "grad_norm": 3.593971777115038,
+      "learning_rate": 0.00016302083333333332,
+      "loss": 0.5836,
+      "step": 314
+    },
+    {
+      "epoch": 0.984375,
+      "grad_norm": 4.287283690498829,
+      "learning_rate": 0.00016354166666666668,
+      "loss": 0.6399,
+      "step": 315
+    },
+    {
+      "epoch": 0.9875,
+      "grad_norm": 2.9784776593955806,
+      "learning_rate": 0.0001640625,
+      "loss": 0.5512,
+      "step": 316
+    },
+    {
+      "epoch": 0.990625,
+      "grad_norm": 5.564221306406586,
+      "learning_rate": 0.00016458333333333334,
+      "loss": 0.6978,
+      "step": 317
+    },
+    {
+      "epoch": 0.99375,
+      "grad_norm": 4.42395876676637,
+      "learning_rate": 0.00016510416666666667,
+      "loss": 0.6639,
+      "step": 318
+    },
+    {
+      "epoch": 0.996875,
+      "grad_norm": 4.140311353469645,
+      "learning_rate": 0.000165625,
+      "loss": 0.6476,
+      "step": 319
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 4.561580859386521,
+      "learning_rate": 0.00016614583333333333,
+      "loss": 0.6309,
+      "step": 320
+    },
+    {
+      "epoch": 1.003125,
+      "grad_norm": 4.434609821187959,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 0.5968,
+      "step": 321
+    },
+    {
+      "epoch": 1.00625,
+      "grad_norm": 4.033931087729723,
+      "learning_rate": 0.00016718750000000002,
+      "loss": 0.612,
+      "step": 322
+    },
+    {
+      "epoch": 1.009375,
+      "grad_norm": 3.668618116153994,
+      "learning_rate": 0.00016770833333333332,
+      "loss": 0.5852,
+      "step": 323
+    },
+    {
+      "epoch": 1.0125,
+      "grad_norm": 3.303816620597269,
+      "learning_rate": 0.00016822916666666668,
+      "loss": 0.582,
+      "step": 324
+    },
+    {
+      "epoch": 1.015625,
+      "grad_norm": 5.842683135889451,
+      "learning_rate": 0.00016875,
+      "loss": 0.6734,
+      "step": 325
+    },
+    {
+      "epoch": 1.01875,
+      "grad_norm": 4.929416158829141,
+      "learning_rate": 0.00016927083333333334,
+      "loss": 0.6835,
+      "step": 326
+    },
+    {
+      "epoch": 1.021875,
+      "grad_norm": 6.063302192931308,
+      "learning_rate": 0.00016979166666666667,
+      "loss": 0.6708,
+      "step": 327
+    },
+    {
+      "epoch": 1.025,
+      "grad_norm": 4.6947744345482185,
+      "learning_rate": 0.0001703125,
+      "loss": 0.5882,
+      "step": 328
+    },
+    {
+      "epoch": 1.028125,
+      "grad_norm": 6.192987624369097,
+      "learning_rate": 0.00017083333333333333,
+      "loss": 0.6155,
+      "step": 329
+    },
+    {
+      "epoch": 1.03125,
+      "grad_norm": 5.968012128174232,
+      "learning_rate": 0.0001713541666666667,
+      "loss": 0.7291,
+      "step": 330
+    },
+    {
+      "epoch": 1.034375,
+      "grad_norm": 4.132939310548836,
+      "learning_rate": 0.00017187500000000002,
+      "loss": 0.572,
+      "step": 331
+    },
+    {
+      "epoch": 1.0375,
+      "grad_norm": 3.960470052640584,
+      "learning_rate": 0.00017239583333333335,
+      "loss": 0.5826,
+      "step": 332
+    },
+    {
+      "epoch": 1.040625,
+      "grad_norm": 3.9920661904821557,
+      "learning_rate": 0.00017291666666666668,
+      "loss": 0.5968,
+      "step": 333
+    },
+    {
+      "epoch": 1.04375,
+      "grad_norm": 3.585872072815213,
+      "learning_rate": 0.0001734375,
+      "loss": 0.5658,
+      "step": 334
+    },
+    {
+      "epoch": 1.046875,
+      "grad_norm": 4.504950881201075,
+      "learning_rate": 0.00017395833333333334,
+      "loss": 0.6037,
+      "step": 335
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 3.1469345496615584,
+      "learning_rate": 0.0001744791666666667,
+      "loss": 0.6137,
+      "step": 336
+    },
+    {
+      "epoch": 1.053125,
+      "grad_norm": 5.747219318871885,
+      "learning_rate": 0.000175,
+      "loss": 0.6002,
+      "step": 337
+    },
+    {
+      "epoch": 1.05625,
+      "grad_norm": 4.520617585503134,
+      "learning_rate": 0.00017552083333333333,
+      "loss": 0.6405,
+      "step": 338
+    },
+    {
+      "epoch": 1.059375,
+      "grad_norm": 4.569181141830411,
+      "learning_rate": 0.00017604166666666669,
+      "loss": 0.5438,
+      "step": 339
+    },
+    {
+      "epoch": 1.0625,
+      "grad_norm": 3.9784246978308655,
+      "learning_rate": 0.00017656250000000002,
+      "loss": 0.6246,
+      "step": 340
+    },
+    {
+      "epoch": 1.065625,
+      "grad_norm": 3.8001801824838353,
+      "learning_rate": 0.00017708333333333335,
+      "loss": 0.5996,
+      "step": 341
+    },
+    {
+      "epoch": 1.06875,
+      "grad_norm": 3.034079840363021,
+      "learning_rate": 0.00017760416666666668,
+      "loss": 0.5843,
+      "step": 342
+    },
+    {
+      "epoch": 1.071875,
+      "grad_norm": 3.876331687428292,
+      "learning_rate": 0.000178125,
+      "loss": 0.615,
+      "step": 343
+    },
+    {
+      "epoch": 1.075,
+      "grad_norm": 4.635798079642919,
+      "learning_rate": 0.00017864583333333334,
+      "loss": 0.6124,
+      "step": 344
+    },
+    {
+      "epoch": 1.078125,
+      "grad_norm": 2.335432423002044,
+      "learning_rate": 0.0001791666666666667,
+      "loss": 0.5488,
+      "step": 345
+    },
+    {
+      "epoch": 1.08125,
+      "grad_norm": 4.785942622491517,
+      "learning_rate": 0.0001796875,
+      "loss": 0.5998,
+      "step": 346
+    },
+    {
+      "epoch": 1.084375,
+      "grad_norm": 3.0759621265468136,
+      "learning_rate": 0.00018020833333333333,
+      "loss": 0.6086,
+      "step": 347
+    },
+    {
+      "epoch": 1.0875,
+      "grad_norm": 4.2039670911038645,
+      "learning_rate": 0.00018072916666666668,
+      "loss": 0.5477,
+      "step": 348
+    },
+    {
+      "epoch": 1.090625,
+      "grad_norm": 4.027958185916663,
+      "learning_rate": 0.00018125000000000001,
+      "loss": 0.6339,
+      "step": 349
+    },
+    {
+      "epoch": 1.09375,
+      "grad_norm": 2.862234750388971,
+      "learning_rate": 0.00018177083333333334,
+      "loss": 0.572,
+      "step": 350
+    },
+    {
+      "epoch": 1.096875,
+      "grad_norm": 5.757871063053227,
+      "learning_rate": 0.00018229166666666667,
+      "loss": 0.6529,
+      "step": 351
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 4.099736087688541,
+      "learning_rate": 0.0001828125,
+      "loss": 0.6555,
+      "step": 352
+    },
+    {
+      "epoch": 1.103125,
+      "grad_norm": 5.460147996487453,
+      "learning_rate": 0.00018333333333333334,
+      "loss": 0.6182,
+      "step": 353
+    },
+    {
+      "epoch": 1.10625,
+      "grad_norm": 4.914741003135554,
+      "learning_rate": 0.0001838541666666667,
+      "loss": 0.627,
+      "step": 354
+    },
+    {
+      "epoch": 1.109375,
+      "grad_norm": 5.631606820737119,
+      "learning_rate": 0.000184375,
+      "loss": 0.6217,
+      "step": 355
+    },
+    {
+      "epoch": 1.1125,
+      "grad_norm": 4.180371014207427,
+      "learning_rate": 0.00018489583333333333,
+      "loss": 0.6211,
+      "step": 356
+    },
+    {
+      "epoch": 1.115625,
+      "grad_norm": 6.29508654977305,
+      "learning_rate": 0.00018541666666666668,
+      "loss": 0.6882,
+      "step": 357
+    },
+    {
+      "epoch": 1.11875,
+      "grad_norm": 5.623785702087407,
+      "learning_rate": 0.0001859375,
+      "loss": 0.714,
+      "step": 358
+    },
+    {
+      "epoch": 1.121875,
+      "grad_norm": 4.033768210280031,
+      "learning_rate": 0.00018645833333333334,
+      "loss": 0.5804,
+      "step": 359
+    },
+    {
+      "epoch": 1.125,
+      "grad_norm": 5.4054964903748015,
+      "learning_rate": 0.00018697916666666667,
+      "loss": 0.6733,
+      "step": 360
+    },
+    {
+      "epoch": 1.128125,
+      "grad_norm": 3.2916387927953217,
+      "learning_rate": 0.0001875,
+      "loss": 0.6227,
+      "step": 361
+    },
+    {
+      "epoch": 1.13125,
+      "grad_norm": 6.953763180473268,
+      "learning_rate": 0.00018802083333333333,
+      "loss": 0.6485,
+      "step": 362
+    },
+    {
+      "epoch": 1.134375,
+      "grad_norm": 5.144823067162498,
+      "learning_rate": 0.0001885416666666667,
+      "loss": 0.6367,
+      "step": 363
+    },
+    {
+      "epoch": 1.1375,
+      "grad_norm": 6.345477212391719,
+      "learning_rate": 0.00018906250000000002,
+      "loss": 0.679,
+      "step": 364
+    },
+    {
+      "epoch": 1.140625,
+      "grad_norm": 5.521239313714407,
+      "learning_rate": 0.00018958333333333332,
+      "loss": 0.6169,
+      "step": 365
+    },
+    {
+      "epoch": 1.14375,
+      "grad_norm": 5.919307096024343,
+      "learning_rate": 0.00019010416666666668,
+      "loss": 0.6229,
+      "step": 366
+    },
+    {
+      "epoch": 1.146875,
+      "grad_norm": 4.767283639783667,
+      "learning_rate": 0.000190625,
+      "loss": 0.6573,
+      "step": 367
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 5.697884009205893,
+      "learning_rate": 0.00019114583333333334,
+      "loss": 0.6571,
+      "step": 368
+    },
+    {
+      "epoch": 1.153125,
+      "grad_norm": 4.448168254264531,
+      "learning_rate": 0.00019166666666666667,
+      "loss": 0.6302,
+      "step": 369
+    },
+    {
+      "epoch": 1.15625,
+      "grad_norm": 4.642921977216857,
+      "learning_rate": 0.0001921875,
+      "loss": 0.5758,
+      "step": 370
+    },
+    {
+      "epoch": 1.159375,
+      "grad_norm": 4.213613761616212,
+      "learning_rate": 0.00019270833333333333,
+      "loss": 0.6153,
+      "step": 371
+    },
+    {
+      "epoch": 1.1625,
+      "grad_norm": 4.149635364368779,
+      "learning_rate": 0.0001932291666666667,
+      "loss": 0.6129,
+      "step": 372
+    },
+    {
+      "epoch": 1.165625,
+      "grad_norm": 3.720503855815432,
+      "learning_rate": 0.00019375000000000002,
+      "loss": 0.622,
+      "step": 373
+    },
+    {
+      "epoch": 1.16875,
+      "grad_norm": 4.545708715962929,
+      "learning_rate": 0.00019427083333333332,
+      "loss": 0.5517,
+      "step": 374
+    },
+    {
+      "epoch": 1.171875,
+      "grad_norm": 3.581342691314205,
+      "learning_rate": 0.00019479166666666668,
+      "loss": 0.5441,
+      "step": 375
+    },
+    {
+      "epoch": 1.175,
+      "grad_norm": 3.6231310988073524,
+      "learning_rate": 0.0001953125,
+      "loss": 0.5731,
+      "step": 376
+    },
+    {
+      "epoch": 1.178125,
+      "grad_norm": 3.0983955669977594,
+      "learning_rate": 0.00019583333333333334,
+      "loss": 0.5655,
+      "step": 377
+    },
+    {
+      "epoch": 1.18125,
+      "grad_norm": 3.3190691945194826,
+      "learning_rate": 0.00019635416666666667,
+      "loss": 0.5668,
+      "step": 378
+    },
+    {
+      "epoch": 1.184375,
+      "grad_norm": 3.4501535392004556,
+      "learning_rate": 0.000196875,
+      "loss": 0.5884,
+      "step": 379
+    },
+    {
+      "epoch": 1.1875,
+      "grad_norm": 3.0103578703577876,
+      "learning_rate": 0.00019739583333333333,
+      "loss": 0.5464,
+      "step": 380
+    },
+    {
+      "epoch": 1.190625,
+      "grad_norm": 4.042886233808065,
+      "learning_rate": 0.0001979166666666667,
+      "loss": 0.5546,
+      "step": 381
+    },
+    {
+      "epoch": 1.19375,
+      "grad_norm": 2.5784932247763,
+      "learning_rate": 0.00019843750000000002,
+      "loss": 0.5198,
+      "step": 382
+    },
+    {
+      "epoch": 1.196875,
+      "grad_norm": 5.067105876382611,
+      "learning_rate": 0.00019895833333333332,
+      "loss": 0.5932,
+      "step": 383
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 2.9767294457122655,
+      "learning_rate": 0.00019947916666666668,
+      "loss": 0.5017,
+      "step": 384
+    },
+    {
+      "epoch": 1.203125,
+      "grad_norm": 4.8940852716921,
+      "learning_rate": 0.0002,
+      "loss": 0.5745,
+      "step": 385
+    },
+    {
+      "epoch": 1.20625,
+      "grad_norm": 4.333946023984807,
+      "learning_rate": 0.0001999999967988477,
+      "loss": 0.616,
+      "step": 386
+    },
+    {
+      "epoch": 1.209375,
+      "grad_norm": 5.093976652912978,
+      "learning_rate": 0.00019999998719539094,
+      "loss": 0.6451,
+      "step": 387
+    },
+    {
+      "epoch": 1.2125,
+      "grad_norm": 4.2881978831171415,
+      "learning_rate": 0.00019999997118963037,
+      "loss": 0.6015,
+      "step": 388
+    },
+    {
+      "epoch": 1.215625,
+      "grad_norm": 4.7565437026635955,
+      "learning_rate": 0.000199999948781567,
+      "loss": 0.5807,
+      "step": 389
+    },
+    {
+      "epoch": 1.21875,
+      "grad_norm": 3.8014460966358823,
+      "learning_rate": 0.00019999991997120227,
+      "loss": 0.5946,
+      "step": 390
+    },
+    {
+      "epoch": 1.221875,
+      "grad_norm": 4.266773944541441,
+      "learning_rate": 0.00019999988475853802,
+      "loss": 0.5801,
+      "step": 391
+    },
+    {
+      "epoch": 1.225,
+      "grad_norm": 3.212291016698606,
+      "learning_rate": 0.00019999984314357652,
+      "loss": 0.5527,
+      "step": 392
+    },
+    {
+      "epoch": 1.228125,
+      "grad_norm": 4.552307155908715,
+      "learning_rate": 0.0001999997951263204,
+      "loss": 0.643,
+      "step": 393
+    },
+    {
+      "epoch": 1.23125,
+      "grad_norm": 3.3806578652171155,
+      "learning_rate": 0.00019999974070677277,
+      "loss": 0.593,
+      "step": 394
+    },
+    {
+      "epoch": 1.234375,
+      "grad_norm": 3.4233889432997295,
+      "learning_rate": 0.0001999996798849371,
+      "loss": 0.5769,
+      "step": 395
+    },
+    {
+      "epoch": 1.2375,
+      "grad_norm": 2.8124449711658137,
+      "learning_rate": 0.0001999996126608173,
+      "loss": 0.5415,
+      "step": 396
+    },
+    {
+      "epoch": 1.240625,
+      "grad_norm": 3.625572686591024,
+      "learning_rate": 0.00019999953903441763,
+      "loss": 0.5659,
+      "step": 397
+    },
+    {
+      "epoch": 1.24375,
+      "grad_norm": 2.3227203649344004,
+      "learning_rate": 0.00019999945900574289,
+      "loss": 0.5364,
+      "step": 398
+    },
+    {
+      "epoch": 1.246875,
+      "grad_norm": 5.252930551183527,
+      "learning_rate": 0.00019999937257479807,
+      "loss": 0.578,
+      "step": 399
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 4.3721674508660096,
+      "learning_rate": 0.00019999927974158883,
+      "loss": 0.587,
+      "step": 400
+    },
+    {
+      "epoch": 1.253125,
+      "grad_norm": 3.1202241697706565,
+      "learning_rate": 0.00019999918050612108,
+      "loss": 0.5307,
+      "step": 401
+    },
+    {
+      "epoch": 1.25625,
+      "grad_norm": 4.029798825672641,
+      "learning_rate": 0.00019999907486840113,
+      "loss": 0.5431,
+      "step": 402
+    },
+    {
+      "epoch": 1.259375,
+      "grad_norm": 2.9257069930809187,
+      "learning_rate": 0.00019999896282843575,
+      "loss": 0.5401,
+      "step": 403
+    },
+    {
+      "epoch": 1.2625,
+      "grad_norm": 3.911075928594739,
+      "learning_rate": 0.00019999884438623216,
+      "loss": 0.5101,
+      "step": 404
+    },
+    {
+      "epoch": 1.265625,
+      "grad_norm": 3.0992066994896583,
+      "learning_rate": 0.00019999871954179792,
+      "loss": 0.5742,
+      "step": 405
+    },
+    {
+      "epoch": 1.26875,
+      "grad_norm": 3.5805638013032732,
+      "learning_rate": 0.000199998588295141,
+      "loss": 0.5687,
+      "step": 406
+    },
+    {
+      "epoch": 1.271875,
+      "grad_norm": 2.681444363924163,
+      "learning_rate": 0.00019999845064626982,
+      "loss": 0.5717,
+      "step": 407
+    },
+    {
+      "epoch": 1.275,
+      "grad_norm": 2.7972868007210767,
+      "learning_rate": 0.0001999983065951932,
+      "loss": 0.5501,
+      "step": 408
+    },
+    {
+      "epoch": 1.278125,
+      "grad_norm": 4.085054846207015,
+      "learning_rate": 0.00019999815614192036,
+      "loss": 0.5172,
+      "step": 409
+    },
+    {
+      "epoch": 1.28125,
+      "grad_norm": 2.693152743035582,
+      "learning_rate": 0.00019999799928646094,
+      "loss": 0.552,
+      "step": 410
+    },
+    {
+      "epoch": 1.284375,
+      "grad_norm": 4.543906959537988,
+      "learning_rate": 0.00019999783602882496,
+      "loss": 0.6084,
+      "step": 411
+    },
+    {
+      "epoch": 1.2875,
+      "grad_norm": 2.883724076312403,
+      "learning_rate": 0.00019999766636902288,
+      "loss": 0.6001,
+      "step": 412
+    },
+    {
+      "epoch": 1.290625,
+      "grad_norm": 3.9635490001420015,
+      "learning_rate": 0.0001999974903070656,
+      "loss": 0.5454,
+      "step": 413
+    },
+    {
+      "epoch": 1.29375,
+      "grad_norm": 2.731161973299382,
+      "learning_rate": 0.00019999730784296433,
+      "loss": 0.5728,
+      "step": 414
+    },
+    {
+      "epoch": 1.296875,
+      "grad_norm": 5.468166004466353,
+      "learning_rate": 0.0001999971189767308,
+      "loss": 0.6353,
+      "step": 415
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 3.195340206239341,
+      "learning_rate": 0.00019999692370837707,
+      "loss": 0.5856,
+      "step": 416
+    },
+    {
+      "epoch": 1.303125,
+      "grad_norm": 5.623783576070044,
+      "learning_rate": 0.00019999672203791565,
+      "loss": 0.5977,
+      "step": 417
+    },
+    {
+      "epoch": 1.30625,
+      "grad_norm": 4.692322670510212,
+      "learning_rate": 0.00019999651396535945,
+      "loss": 0.5914,
+      "step": 418
+    },
+    {
+      "epoch": 1.309375,
+      "grad_norm": 3.954568184680778,
+      "learning_rate": 0.0001999962994907218,
+      "loss": 0.5213,
+      "step": 419
+    },
+    {
+      "epoch": 1.3125,
+      "grad_norm": 4.136861297309464,
+      "learning_rate": 0.00019999607861401645,
+      "loss": 0.5308,
+      "step": 420
+    },
+    {
+      "epoch": 1.315625,
+      "grad_norm": 3.615770213787855,
+      "learning_rate": 0.00019999585133525746,
+      "loss": 0.5723,
+      "step": 421
+    },
+    {
+      "epoch": 1.31875,
+      "grad_norm": 2.9300725255491367,
+      "learning_rate": 0.00019999561765445952,
+      "loss": 0.5139,
+      "step": 422
+    },
+    {
+      "epoch": 1.321875,
+      "grad_norm": 3.348457829733626,
+      "learning_rate": 0.00019999537757163746,
+      "loss": 0.5599,
+      "step": 423
+    },
+    {
+      "epoch": 1.325,
+      "grad_norm": 3.4438568677902017,
+      "learning_rate": 0.00019999513108680673,
+      "loss": 0.5402,
+      "step": 424
+    },
+    {
+      "epoch": 1.328125,
+      "grad_norm": 3.8973380329575273,
+      "learning_rate": 0.00019999487819998305,
+      "loss": 0.6013,
+      "step": 425
+    },
+    {
+      "epoch": 1.33125,
+      "grad_norm": 2.544244396987993,
+      "learning_rate": 0.00019999461891118266,
+      "loss": 0.5641,
+      "step": 426
+    },
+    {
+      "epoch": 1.334375,
+      "grad_norm": 3.8790631885370894,
+      "learning_rate": 0.00019999435322042217,
+      "loss": 0.5024,
+      "step": 427
+    },
+    {
+      "epoch": 1.3375,
+      "grad_norm": 3.7697598697037527,
+      "learning_rate": 0.00019999408112771853,
+      "loss": 0.5722,
+      "step": 428
+    },
+    {
+      "epoch": 1.340625,
+      "grad_norm": 2.839502929655522,
+      "learning_rate": 0.0001999938026330892,
+      "loss": 0.6027,
+      "step": 429
+    },
+    {
+      "epoch": 1.34375,
+      "grad_norm": 3.45233079645525,
+      "learning_rate": 0.00019999351773655204,
+      "loss": 0.5905,
+      "step": 430
+    },
+    {
+      "epoch": 1.346875,
+      "grad_norm": 3.621918441298716,
+      "learning_rate": 0.0001999932264381252,
+      "loss": 0.5688,
+      "step": 431
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 2.348120809505185,
+      "learning_rate": 0.0001999929287378274,
+      "loss": 0.5413,
+      "step": 432
+    },
+    {
+      "epoch": 1.353125,
+      "grad_norm": 4.08459549931994,
+      "learning_rate": 0.00019999262463567773,
+      "loss": 0.5694,
+      "step": 433
+    },
+    {
+      "epoch": 1.35625,
+      "grad_norm": 2.6346713908504023,
+      "learning_rate": 0.00019999231413169556,
+      "loss": 0.543,
+      "step": 434
+    },
+    {
+      "epoch": 1.359375,
+      "grad_norm": 3.3167096234149964,
+      "learning_rate": 0.00019999199722590085,
+      "loss": 0.5308,
+      "step": 435
+    },
+    {
+      "epoch": 1.3625,
+      "grad_norm": 2.6791404002072032,
+      "learning_rate": 0.00019999167391831387,
+      "loss": 0.5298,
+      "step": 436
+    },
+    {
+      "epoch": 1.365625,
+      "grad_norm": 3.4841203686972757,
+      "learning_rate": 0.0001999913442089553,
+      "loss": 0.5679,
+      "step": 437
+    },
+    {
+      "epoch": 1.36875,
+      "grad_norm": 2.9536436274783275,
+      "learning_rate": 0.00019999100809784624,
+      "loss": 0.5635,
+      "step": 438
+    },
+    {
+      "epoch": 1.371875,
+      "grad_norm": 2.835194073224616,
+      "learning_rate": 0.00019999066558500827,
+      "loss": 0.4898,
+      "step": 439
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 2.781774587646673,
+      "learning_rate": 0.00019999031667046322,
+      "loss": 0.5007,
+      "step": 440
+    },
+    {
+      "epoch": 1.378125,
+      "grad_norm": 2.208584429068845,
+      "learning_rate": 0.00019998996135423352,
+      "loss": 0.5729,
+      "step": 441
+    },
+    {
+      "epoch": 1.38125,
+      "grad_norm": 2.2587000899586274,
+      "learning_rate": 0.0001999895996363419,
+      "loss": 0.5187,
+      "step": 442
+    },
+    {
+      "epoch": 1.384375,
+      "grad_norm": 2.3952388352439122,
+      "learning_rate": 0.00019998923151681147,
+      "loss": 0.4604,
+      "step": 443
+    },
+    {
+      "epoch": 1.3875,
+      "grad_norm": 2.3986265537515337,
+      "learning_rate": 0.00019998885699566588,
+      "loss": 0.5013,
+      "step": 444
+    },
+    {
+      "epoch": 1.390625,
+      "grad_norm": 3.925642267628311,
+      "learning_rate": 0.000199988476072929,
+      "loss": 0.516,
+      "step": 445
+    },
+    {
+      "epoch": 1.39375,
+      "grad_norm": 1.8832343973114467,
+      "learning_rate": 0.00019998808874862532,
+      "loss": 0.5272,
+      "step": 446
+    },
+    {
+      "epoch": 1.396875,
+      "grad_norm": 4.692073484437634,
+      "learning_rate": 0.0001999876950227796,
+      "loss": 0.5895,
+      "step": 447
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 2.7189004688911385,
+      "learning_rate": 0.00019998729489541703,
+      "loss": 0.5114,
+      "step": 448
+    },
+    {
+      "epoch": 1.403125,
+      "grad_norm": 5.458041477832237,
+      "learning_rate": 0.00019998688836656323,
+      "loss": 0.6303,
+      "step": 449
+    },
+    {
+      "epoch": 1.40625,
+      "grad_norm": 3.8740132507053353,
+      "learning_rate": 0.00019998647543624425,
+      "loss": 0.5708,
+      "step": 450
+    },
+    {
+      "epoch": 1.409375,
+      "grad_norm": 4.663852862102554,
+      "learning_rate": 0.0001999860561044865,
+      "loss": 0.5054,
+      "step": 451
+    },
+    {
+      "epoch": 1.4125,
+      "grad_norm": 4.236208251222634,
+      "learning_rate": 0.00019998563037131682,
+      "loss": 0.6268,
+      "step": 452
+    },
+    {
+      "epoch": 1.415625,
+      "grad_norm": 3.524886794904158,
+      "learning_rate": 0.00019998519823676253,
+      "loss": 0.5446,
+      "step": 453
+    },
+    {
+      "epoch": 1.41875,
+      "grad_norm": 3.653003723788189,
+      "learning_rate": 0.00019998475970085125,
+      "loss": 0.5566,
+      "step": 454
+    },
+    {
+      "epoch": 1.421875,
+      "grad_norm": 3.133791650542882,
+      "learning_rate": 0.00019998431476361106,
+      "loss": 0.5152,
+      "step": 455
+    },
+    {
+      "epoch": 1.425,
+      "grad_norm": 2.5802764294356706,
+      "learning_rate": 0.00019998386342507044,
+      "loss": 0.4878,
+      "step": 456
+    },
+    {
+      "epoch": 1.428125,
+      "grad_norm": 3.656491891052452,
+      "learning_rate": 0.0001999834056852583,
+      "loss": 0.6039,
+      "step": 457
+    },
+    {
+      "epoch": 1.43125,
+      "grad_norm": 3.689518376360162,
+      "learning_rate": 0.00019998294154420393,
+      "loss": 0.5489,
+      "step": 458
+    },
+    {
+      "epoch": 1.434375,
+      "grad_norm": 3.0169402432450254,
+      "learning_rate": 0.00019998247100193702,
+      "loss": 0.4919,
+      "step": 459
+    },
+    {
+      "epoch": 1.4375,
+      "grad_norm": 2.2947838041021904,
+      "learning_rate": 0.0001999819940584878,
+      "loss": 0.4612,
+      "step": 460
+    },
+    {
+      "epoch": 1.440625,
+      "grad_norm": 4.3157378953925285,
+      "learning_rate": 0.00019998151071388672,
+      "loss": 0.5555,
+      "step": 461
+    },
+    {
+      "epoch": 1.44375,
+      "grad_norm": 3.331016514811397,
+      "learning_rate": 0.00019998102096816469,
+      "loss": 0.5527,
+      "step": 462
+    },
+    {
+      "epoch": 1.446875,
+      "grad_norm": 3.3099077873032345,
+      "learning_rate": 0.00019998052482135314,
+      "loss": 0.4998,
+      "step": 463
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 3.1142823326754874,
+      "learning_rate": 0.00019998002227348383,
+      "loss": 0.5235,
+      "step": 464
+    },
+    {
+      "epoch": 1.453125,
+      "grad_norm": 3.9524533988710764,
+      "learning_rate": 0.0001999795133245889,
+      "loss": 0.5886,
+      "step": 465
+    },
+    {
+      "epoch": 1.45625,
+      "grad_norm": 3.541960662228808,
+      "learning_rate": 0.00019997899797470097,
+      "loss": 0.4846,
+      "step": 466
+    },
+    {
+      "epoch": 1.459375,
+      "grad_norm": 2.985533474202037,
+      "learning_rate": 0.00019997847622385298,
+      "loss": 0.574,
+      "step": 467
+    },
+    {
+      "epoch": 1.4625,
+      "grad_norm": 3.9207440738080512,
+      "learning_rate": 0.0001999779480720784,
+      "loss": 0.5594,
+      "step": 468
+    },
+    {
+      "epoch": 1.465625,
+      "grad_norm": 2.400502411740158,
+      "learning_rate": 0.00019997741351941101,
+      "loss": 0.5113,
+      "step": 469
+    },
+    {
+      "epoch": 1.46875,
+      "grad_norm": 4.155918440406712,
+      "learning_rate": 0.000199976872565885,
+      "loss": 0.5631,
+      "step": 470
+    },
+    {
+      "epoch": 1.471875,
+      "grad_norm": 4.3228454360372535,
+      "learning_rate": 0.0001999763252115351,
+      "loss": 0.6143,
+      "step": 471
+    },
+    {
+      "epoch": 1.475,
+      "grad_norm": 3.1428945311392207,
+      "learning_rate": 0.00019997577145639626,
+      "loss": 0.4787,
+      "step": 472
+    },
+    {
+      "epoch": 1.478125,
+      "grad_norm": 2.9261070905039683,
+      "learning_rate": 0.00019997521130050394,
+      "loss": 0.5132,
+      "step": 473
+    },
+    {
+      "epoch": 1.48125,
+      "grad_norm": 3.4125204633658903,
+      "learning_rate": 0.00019997464474389408,
+      "loss": 0.5668,
+      "step": 474
+    },
+    {
+      "epoch": 1.484375,
+      "grad_norm": 3.0614394470324786,
+      "learning_rate": 0.00019997407178660285,
+      "loss": 0.5237,
+      "step": 475
+    },
+    {
+      "epoch": 1.4875,
+      "grad_norm": 2.7647602283996027,
+      "learning_rate": 0.000199973492428667,
+      "loss": 0.4709,
+      "step": 476
+    },
+    {
+      "epoch": 1.490625,
+      "grad_norm": 3.4428009587630513,
+      "learning_rate": 0.0001999729066701236,
+      "loss": 0.5353,
+      "step": 477
+    },
+    {
+      "epoch": 1.49375,
+      "grad_norm": 2.7375056626957646,
+      "learning_rate": 0.00019997231451101017,
+      "loss": 0.5056,
+      "step": 478
+    },
+    {
+      "epoch": 1.496875,
+      "grad_norm": 3.2712438843146665,
+      "learning_rate": 0.00019997171595136464,
+      "loss": 0.4901,
+      "step": 479
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 2.6104083009852617,
+      "learning_rate": 0.00019997111099122528,
+      "loss": 0.5733,
+      "step": 480
+    },
+    {
+      "epoch": 1.503125,
+      "grad_norm": 2.956546548553359,
+      "learning_rate": 0.0001999704996306308,
+      "loss": 0.5422,
+      "step": 481
+    },
+    {
+      "epoch": 1.50625,
+      "grad_norm": 2.0881630180588417,
+      "learning_rate": 0.00019996988186962041,
+      "loss": 0.4988,
+      "step": 482
+    },
+    {
+      "epoch": 1.509375,
+      "grad_norm": 3.1962973710170073,
+      "learning_rate": 0.00019996925770823367,
+      "loss": 0.5283,
+      "step": 483
+    },
+    {
+      "epoch": 1.5125,
+      "grad_norm": 2.454784996915068,
+      "learning_rate": 0.00019996862714651047,
+      "loss": 0.485,
+      "step": 484
+    },
+    {
+      "epoch": 1.515625,
+      "grad_norm": 2.6543098898421307,
+      "learning_rate": 0.0001999679901844912,
+      "loss": 0.5362,
+      "step": 485
+    },
+    {
+      "epoch": 1.51875,
+      "grad_norm": 2.6734071676562166,
+      "learning_rate": 0.0001999673468222167,
+      "loss": 0.4973,
+      "step": 486
+    },
+    {
+      "epoch": 1.521875,
+      "grad_norm": 3.5617138387287937,
+      "learning_rate": 0.0001999666970597281,
+      "loss": 0.4839,
+      "step": 487
+    },
+    {
+      "epoch": 1.525,
+      "grad_norm": 2.40337813243574,
+      "learning_rate": 0.00019996604089706703,
+      "loss": 0.47,
+      "step": 488
+    },
+    {
+      "epoch": 1.528125,
+      "grad_norm": 4.077643473345593,
+      "learning_rate": 0.00019996537833427548,
+      "loss": 0.5522,
+      "step": 489
+    },
+    {
+      "epoch": 1.53125,
+      "grad_norm": 2.374710238948673,
+      "learning_rate": 0.00019996470937139584,
+      "loss": 0.5001,
+      "step": 490
+    },
+    {
+      "epoch": 1.534375,
+      "grad_norm": 3.5332391386112043,
+      "learning_rate": 0.00019996403400847102,
+      "loss": 0.471,
+      "step": 491
+    },
+    {
+      "epoch": 1.5375,
+      "grad_norm": 3.464037276431159,
+      "learning_rate": 0.0001999633522455442,
+      "loss": 0.5012,
+      "step": 492
+    },
+    {
+      "epoch": 1.540625,
+      "grad_norm": 2.8912087075191533,
+      "learning_rate": 0.00019996266408265902,
+      "loss": 0.4912,
+      "step": 493
+    },
+    {
+      "epoch": 1.54375,
+      "grad_norm": 2.433207544384389,
+      "learning_rate": 0.00019996196951985957,
+      "loss": 0.5065,
+      "step": 494
+    },
+    {
+      "epoch": 1.546875,
+      "grad_norm": 2.3767389632937697,
+      "learning_rate": 0.0001999612685571903,
+      "loss": 0.4759,
+      "step": 495
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 2.650960704210614,
+      "learning_rate": 0.0001999605611946961,
+      "loss": 0.493,
+      "step": 496
+    },
+    {
+      "epoch": 1.553125,
+      "grad_norm": 1.8470710846159393,
+      "learning_rate": 0.00019995984743242226,
+      "loss": 0.4616,
+      "step": 497
+    },
+    {
+      "epoch": 1.55625,
+      "grad_norm": 2.8761005519647656,
+      "learning_rate": 0.0001999591272704145,
+      "loss": 0.5349,
+      "step": 498
+    },
+    {
+      "epoch": 1.559375,
+      "grad_norm": 2.136684126309438,
+      "learning_rate": 0.00019995840070871884,
+      "loss": 0.4849,
+      "step": 499
+    },
+    {
+      "epoch": 1.5625,
+      "grad_norm": 3.1258227469404387,
+      "learning_rate": 0.00019995766774738187,
+      "loss": 0.4858,
+      "step": 500
+    },
+    {
+      "epoch": 1.5625,
+      "eval_loss": 0.11392344534397125,
+      "eval_runtime": 179.6799,
+      "eval_samples_per_second": 23.653,
+      "eval_steps_per_second": 0.373,
+      "step": 500
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 12800,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 40,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9fc8bca2a96d3955549f3cd9fc6f3e71444bf3de9ddb3c1e06920443b0ffde2
+size 8081

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/video_preprocessor_config.json

@@ -0,0 +1,86 @@
+{
+  "_valid_kwargs_names": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device",
+    "min_pixels",
+    "max_pixels",
+    "patch_size",
+    "temporal_patch_size",
+    "merge_size"
+  ],
+  "crop_size": null,
+  "data_format": "channels_first",
+  "default_to_square": true,
+  "device": null,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_pad": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "input_data_format": null,
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "model_valid_processing_keys": [
+    "do_convert_rgb",
+    "do_resize",
+    "size",
+    "size_divisor",
+    "default_to_square",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "do_pad",
+    "do_center_crop",
+    "crop_size",
+    "data_format",
+    "input_data_format",
+    "device",
+    "min_pixels",
+    "max_pixels",
+    "patch_size",
+    "temporal_patch_size",
+    "merge_size"
+  ],
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "size_divisor": null,
+  "temporal_patch_size": 2,
+  "video_processor_type": "Qwen2VLVideoProcessor"
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_131850/checkpoint-500/zero_to_fp32.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example:
+#   python zero_to_fp32.py . output_dir/
+#   or
+#   python zero_to_fp32.py . output_dir/ --safe_serialization
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+import gc
+import json
+import numpy as np
+from tqdm import tqdm
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device, weights_only=False)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+    total_files = len(files)
+    state_dicts = []
+    for f in tqdm(files, desc='Loading checkpoint shards'):
+        state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+class GatheredTensor:
+    """
+    A pseudo tensor that collects partitioned weights.
+    It is more memory efficient when there are multiple groups.
+    """
+
+    def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape):
+        self.flat_groups = flat_groups
+        self.flat_groups_offset = flat_groups_offset
+        self.offset = offset
+        self.partitioned_numel = partitioned_numel
+        self.shape = shape
+        self.dtype = self.flat_groups[0][0].dtype
+
+    def contiguous(self):
+        """
+        Merge partitioned weights from flat_groups into a single tensor.
+        """
+        end_idx = self.offset + self.partitioned_numel
+        world_size = len(self.flat_groups)
+        pad_flat_param_chunks = []
+
+        for rank_i in range(world_size):
+            # for each rank, we need to collect weights from related group/groups
+            flat_groups_at_rank_i = self.flat_groups[rank_i]
+            start_group_id = None
+            end_group_id = None
+            for group_id in range(len(self.flat_groups_offset)):
+                if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]:
+                    start_group_id = group_id
+                if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]:
+                    end_group_id = group_id
+                    break
+            # collect weights from related group/groups
+            for group_id in range(start_group_id, end_group_id + 1):
+                flat_tensor = flat_groups_at_rank_i[group_id]
+                start_offset = self.offset - self.flat_groups_offset[group_id]
+                end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id]
+                pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset])
+
+        # collect weights from all ranks
+        pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0)
+        param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous()
+        return param
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size
+
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]]))
+    for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'):
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # memory efficient tensor
+        tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape)
+        state_dict[name] = tensor
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def to_torch_tensor(state_dict, return_empty_tensor=False):
+    """
+    Convert state_dict of GatheredTensor to torch tensor
+    """
+    torch_state_dict = {}
+    converted_tensors = {}
+    for name, tensor in state_dict.items():
+        tensor_id = id(tensor)
+        if tensor_id in converted_tensors:  # shared tensors
+            shared_tensor = torch_state_dict[converted_tensors[tensor_id]]
+            torch_state_dict[name] = shared_tensor
+        else:
+            converted_tensors[tensor_id] = name
+            if return_empty_tensor:
+                torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype)
+            else:
+                torch_state_dict[name] = tensor.contiguous()
+    return torch_state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                             tag=None,
+                                             exclude_frozen_parameters=False,
+                                             lazy_mode=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+        - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient.
+          Convert the pesduo tensor to torch tensor by ``.contiguous()``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    Note: the above usage may not work if your application doesn't have sufficient free CPU memory.
+    You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint. Or you can load state_dict in lazy mode ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu
+        for name, lazy_tensor in state_dict.item():
+            tensor = lazy_tensor.contiguous()  # to cpu
+            print(name, tensor)
+            # del tensor to release memory if it no longer in use
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+    if lazy_mode:
+        return state_dict
+    else:
+        return to_torch_tensor(state_dict)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir,
+                                               output_dir,
+                                               max_shard_size="5GB",
+                                               safe_serialization=False,
+                                               tag=None,
+                                               exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_dir``: directory to the pytorch fp32 state_dict output files
+        - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB
+        - ``safe_serialization``:  whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    # Dependency pre-check
+    if safe_serialization:
+        try:
+            from safetensors.torch import save_file
+        except ImportError:
+            print('If you want to use `safe_serialization`, please `pip install safetensors`')
+            raise
+    if max_shard_size is not None:
+        try:
+            from huggingface_hub import split_torch_state_dict_into_shards
+        except ImportError:
+            print('If you want to use `max_shard_size`, please `pip install huggingface_hub`')
+            raise
+
+    # Convert zero checkpoint to state_dict
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir,
+                                                          tag,
+                                                          exclude_frozen_parameters,
+                                                          lazy_mode=True)
+
+    # Shard the model if it is too big.
+    weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin"
+    if max_shard_size is not None:
+        filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors")
+        # an memory-efficient approach for sharding
+        empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True)
+        state_dict_split = split_torch_state_dict_into_shards(empty_state_dict,
+                                                              filename_pattern=filename_pattern,
+                                                              max_shard_size=max_shard_size)
+    else:
+        from collections import namedtuple
+        StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"])
+        state_dict_split = StateDictSplit(is_sharded=False,
+                                          filename_to_tensors={weights_name: list(state_dict.keys())})
+
+    # Save the model by shard
+    os.makedirs(output_dir, exist_ok=True)
+    filename_to_tensors = state_dict_split.filename_to_tensors.items()
+    for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"):
+        shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors}
+        shard_state_dict = to_torch_tensor(shard_state_dict)
+        output_path = os.path.join(output_dir, shard_file)
+        if safe_serialization:
+            save_file(shard_state_dict, output_path, metadata={"format": "pt"})
+        else:
+            torch.save(shard_state_dict, output_path)
+        # release the memory of current shard
+        for tensor_name in list(shard_state_dict.keys()):
+            del state_dict[tensor_name]
+            del shard_state_dict[tensor_name]
+        del shard_state_dict
+        gc.collect()
+
+    # Save index if sharded
+    if state_dict_split.is_sharded:
+        index = {
+            "metadata": state_dict_split.metadata,
+            "weight_map": state_dict_split.tensor_to_filename,
+        }
+        save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json"
+        save_index_file = os.path.join(output_dir, save_index_file)
+        with open(save_index_file, "w", encoding="utf-8") as f:
+            content = json.dumps(index, indent=2, sort_keys=True) + "\n"
+            f.write(content)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument("output_dir",
+                        type=str,
+                        help="directory to the pytorch fp32 state_dict output files"
+                        "(e.g. path/checkpoint-12-output/)")
+    parser.add_argument(
+        "--max_shard_size",
+        type=str,
+        default="5GB",
+        help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size"
+        "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`"
+        "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances"
+        "without CPU OOM issues.")
+    parser.add_argument(
+        "--safe_serialization",
+        default=False,
+        action='store_true',
+        help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_dir,
+                                               max_shard_size=args.max_shard_size,
+                                               safe_serialization=args.safe_serialization,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9611db4ecaa59d3fc70ceb5b941fad532155fa85cd5aeff43f7720ae88d6d4c
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af859d8c4bf341d92d0486c684d2ef9565fc9bf092b17855fbd073a7180763f4
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dd614f40a553d1bb873908360ba10b2b6867bc3f706fb675838b45b6915b21c
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:053277b58493433d62d59410c450960c06ddb50a36e3290883c4a87ea4602ac5
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a67cd2ba80bf09063988c7e0d1ad7215a2e3bc906c923eb85f0d7d276fc1e64
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6705d09ae69ab1b22a81164c323ee8d652a16426974b73b5fbf7ee6b71903fa
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d61022cbf45faaea271d57a22fd040c4e8afe78e3b3c9c526fafeef3255775ca
+size 344223365

checkpoint/pipeline_0225/StreamingVLM_SFT_stage_1_e40_lr2e-4_ps512_pw512_20260226_190228/checkpoint-1000/global_step1000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e9c5a0fc6e599fdcbd25629498e11dbbeb9ba33396e5df572e998682f59654
+size 344223365