add checkpoints/codi_frozen_1.5b_hidden

.gitattributes

@@ -90,3 +90,7 @@ checkpoints/codi3b_a1.0_b1.0_g1.0_ls2/checkpoint-400/tokenizer.json filter=lfs d
 checkpoints/codi3b_a1.0_b1.0_g1.0_ls2/checkpoint-600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/codi3b_a1.0_b1.0_g1.0_ls2/checkpoint-800/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/codi3b_a1.0_b1.0_g1.0_ls2/wandb/offline-run-20260620_202751-dlf846se/run-dlf846se.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/run-a09es240.wandb filter=lfs diff=lfs merge=lfs -text

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d4742b3b303751e4360ae72847861b17c4d7b9c10813f222f361c116ef7469
+size 3096275515

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/thought_projector.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f3270f574717bc6ad282a7e811ea404899cb309340d48b23246d051ed18a619
+size 9445953

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1000/trainer_state.json

@@ -0,0 +1,2034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.28835063437139563,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014417531718569781,
+      "grad_norm": 322.0,
+      "kd_loss": 1.997420072555542,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 2.1422,
+      "step": 5,
+      "student_loss": 0.0605761781334877,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0028835063437139563,
+      "grad_norm": 186.0,
+      "kd_loss": 1.8595829010009766,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 2.0829,
+      "step": 10,
+      "student_loss": 0.09249057620763779,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.004325259515570935,
+      "grad_norm": 117.5,
+      "kd_loss": 1.5637085437774658,
+      "learning_rate": 1.5555555555555558e-06,
+      "loss": 1.972,
+      "step": 15,
+      "student_loss": 0.026328546926379204,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0057670126874279125,
+      "grad_norm": 106.5,
+      "kd_loss": 1.4912058115005493,
+      "learning_rate": 2.1111111111111114e-06,
+      "loss": 1.8011,
+      "step": 20,
+      "student_loss": 0.13309946656227112,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00720876585928489,
+      "grad_norm": 86.5,
+      "kd_loss": 1.1542646884918213,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 1.5427,
+      "step": 25,
+      "student_loss": 0.036511313170194626,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00865051903114187,
+      "grad_norm": 58.5,
+      "kd_loss": 0.9403514266014099,
+      "learning_rate": 3.2222222222222227e-06,
+      "loss": 1.2794,
+      "step": 30,
+      "student_loss": 0.01801167242228985,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.010092272202998846,
+      "grad_norm": 31.5,
+      "kd_loss": 0.6764306426048279,
+      "learning_rate": 3.777777777777778e-06,
+      "loss": 1.0897,
+      "step": 35,
+      "student_loss": 0.0011638500727713108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.011534025374855825,
+      "grad_norm": 27.625,
+      "kd_loss": 0.7702282667160034,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 0.9024,
+      "step": 40,
+      "student_loss": 0.08332191407680511,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.012975778546712802,
+      "grad_norm": 20.875,
+      "kd_loss": 1.050789475440979,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.7206,
+      "step": 45,
+      "student_loss": 0.07726961374282837,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01441753171856978,
+      "grad_norm": 44.25,
+      "kd_loss": 0.4379478096961975,
+      "learning_rate": 5.444444444444445e-06,
+      "loss": 0.6754,
+      "step": 50,
+      "student_loss": 0.007436605170369148,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.015859284890426758,
+      "grad_norm": 17.5,
+      "kd_loss": 0.3016382157802582,
+      "learning_rate": 6e-06,
+      "loss": 0.5857,
+      "step": 55,
+      "student_loss": 0.002162981079891324,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01730103806228374,
+      "grad_norm": 24.625,
+      "kd_loss": 0.2297545075416565,
+      "learning_rate": 6.555555555555556e-06,
+      "loss": 0.5265,
+      "step": 60,
+      "student_loss": 0.012505847960710526,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.018742791234140715,
+      "grad_norm": 14.75,
+      "kd_loss": 0.2053602635860443,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 0.5101,
+      "step": 65,
+      "student_loss": 0.04744679108262062,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.020184544405997693,
+      "grad_norm": 34.0,
+      "kd_loss": 0.7437167763710022,
+      "learning_rate": 7.666666666666667e-06,
+      "loss": 0.4523,
+      "step": 70,
+      "student_loss": 0.029973715543746948,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02162629757785467,
+      "grad_norm": 14.0625,
+      "kd_loss": 0.18727970123291016,
+      "learning_rate": 8.222222222222222e-06,
+      "loss": 0.4534,
+      "step": 75,
+      "student_loss": 0.001686512609012425,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02306805074971165,
+      "grad_norm": 31.0,
+      "kd_loss": 0.18602637946605682,
+      "learning_rate": 8.777777777777778e-06,
+      "loss": 0.4338,
+      "step": 80,
+      "student_loss": 0.0014192159287631512,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.024509803921568627,
+      "grad_norm": 14.75,
+      "kd_loss": 0.11737873405218124,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.4378,
+      "step": 85,
+      "student_loss": 0.000477277091704309,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.025951557093425604,
+      "grad_norm": 17.125,
+      "kd_loss": 0.7024360299110413,
+      "learning_rate": 9.88888888888889e-06,
+      "loss": 0.3816,
+      "step": 90,
+      "student_loss": 0.027783483266830444,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.027393310265282585,
+      "grad_norm": 19.125,
+      "kd_loss": 0.3916384279727936,
+      "learning_rate": 9.99995337991255e-06,
+      "loss": 0.3905,
+      "step": 95,
+      "student_loss": 0.013136160559952259,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02883506343713956,
+      "grad_norm": 13.9375,
+      "kd_loss": 0.09685378521680832,
+      "learning_rate": 9.999763987297266e-06,
+      "loss": 0.3485,
+      "step": 100,
+      "student_loss": 0.0006464557372964919,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03027681660899654,
+      "grad_norm": 12.5,
+      "kd_loss": 0.48202934861183167,
+      "learning_rate": 9.999428913912876e-06,
+      "loss": 0.3555,
+      "step": 105,
+      "student_loss": 0.005315156653523445,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.031718569780853516,
+      "grad_norm": 9.8125,
+      "kd_loss": 0.42474475502967834,
+      "learning_rate": 9.998948169522587e-06,
+      "loss": 0.3616,
+      "step": 110,
+      "student_loss": 0.0304458886384964,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03316032295271049,
+      "grad_norm": 11.9375,
+      "kd_loss": 0.30219101905822754,
+      "learning_rate": 9.998321768134101e-06,
+      "loss": 0.3669,
+      "step": 115,
+      "student_loss": 0.0007903591031208634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03460207612456748,
+      "grad_norm": 47.0,
+      "kd_loss": 0.14825306832790375,
+      "learning_rate": 9.997549727999212e-06,
+      "loss": 0.3014,
+      "step": 120,
+      "student_loss": 0.0007933085435070097,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.036043829296424454,
+      "grad_norm": 10.6875,
+      "kd_loss": 0.6828317046165466,
+      "learning_rate": 9.996632071613257e-06,
+      "loss": 0.3416,
+      "step": 125,
+      "student_loss": 0.056072596460580826,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03748558246828143,
+      "grad_norm": 21.5,
+      "kd_loss": 0.16518603265285492,
+      "learning_rate": 9.995568825714479e-06,
+      "loss": 0.3623,
+      "step": 130,
+      "student_loss": 0.000613231910392642,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03892733564013841,
+      "grad_norm": 11.875,
+      "kd_loss": 0.4139902889728546,
+      "learning_rate": 9.994360021283235e-06,
+      "loss": 0.3269,
+      "step": 135,
+      "student_loss": 0.059860486537218094,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.040369088811995385,
+      "grad_norm": 17.0,
+      "kd_loss": 0.15953019261360168,
+      "learning_rate": 9.993005693541098e-06,
+      "loss": 0.2964,
+      "step": 140,
+      "student_loss": 0.0005189953953959048,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04181084198385236,
+      "grad_norm": 19.5,
+      "kd_loss": 0.7493298053741455,
+      "learning_rate": 9.991505881949837e-06,
+      "loss": 0.3058,
+      "step": 145,
+      "student_loss": 0.053292322903871536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04325259515570934,
+      "grad_norm": 15.8125,
+      "kd_loss": 0.09782249480485916,
+      "learning_rate": 9.989860630210251e-06,
+      "loss": 0.3046,
+      "step": 150,
+      "student_loss": 0.0007303969468921423,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04469434832756632,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.119688019156456,
+      "learning_rate": 9.98806998626091e-06,
+      "loss": 0.3088,
+      "step": 155,
+      "student_loss": 0.0005232616094872355,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0461361014994233,
+      "grad_norm": 17.625,
+      "kd_loss": 0.2501586973667145,
+      "learning_rate": 9.98613400227676e-06,
+      "loss": 0.3508,
+      "step": 160,
+      "student_loss": 0.0909157246351242,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04757785467128028,
+      "grad_norm": 25.75,
+      "kd_loss": 0.23997722566127777,
+      "learning_rate": 9.984052734667586e-06,
+      "loss": 0.309,
+      "step": 165,
+      "student_loss": 0.0003585258382372558,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.049019607843137254,
+      "grad_norm": 53.75,
+      "kd_loss": 0.08952173590660095,
+      "learning_rate": 9.981826244076387e-06,
+      "loss": 0.2906,
+      "step": 170,
+      "student_loss": 0.0005561846191994846,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05046136101499423,
+      "grad_norm": 40.0,
+      "kd_loss": 0.22894923388957977,
+      "learning_rate": 9.979454595377594e-06,
+      "loss": 0.2992,
+      "step": 175,
+      "student_loss": 0.0037626794073730707,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05190311418685121,
+      "grad_norm": 23.75,
+      "kd_loss": 0.23231007158756256,
+      "learning_rate": 9.97693785767519e-06,
+      "loss": 0.278,
+      "step": 180,
+      "student_loss": 0.0004788874357473105,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05334486735870819,
+      "grad_norm": 32.25,
+      "kd_loss": 0.32399287819862366,
+      "learning_rate": 9.9742761043007e-06,
+      "loss": 0.3037,
+      "step": 185,
+      "student_loss": 0.05988827347755432,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05478662053056517,
+      "grad_norm": 18.75,
+      "kd_loss": 0.46008631587028503,
+      "learning_rate": 9.971469412811032e-06,
+      "loss": 0.2833,
+      "step": 190,
+      "student_loss": 0.0008702761260792613,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.056228373702422146,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.41200897097587585,
+      "learning_rate": 9.968517864986247e-06,
+      "loss": 0.2848,
+      "step": 195,
+      "student_loss": 0.01731002703309059,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05767012687427912,
+      "grad_norm": 10.875,
+      "kd_loss": 0.17150147259235382,
+      "learning_rate": 9.965421546827153e-06,
+      "loss": 0.2805,
+      "step": 200,
+      "student_loss": 0.0014504786813631654,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0591118800461361,
+      "grad_norm": 20.875,
+      "kd_loss": 0.1884479522705078,
+      "learning_rate": 9.962180548552812e-06,
+      "loss": 0.2697,
+      "step": 205,
+      "student_loss": 0.04227042570710182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06055363321799308,
+      "grad_norm": 23.75,
+      "kd_loss": 0.24175123870372772,
+      "learning_rate": 9.958794964597906e-06,
+      "loss": 0.2654,
+      "step": 210,
+      "student_loss": 0.0005445634596981108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.061995386389850055,
+      "grad_norm": 25.875,
+      "kd_loss": 0.2347751259803772,
+      "learning_rate": 9.955264893609986e-06,
+      "loss": 0.294,
+      "step": 215,
+      "student_loss": 0.0005844403640367091,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06343713956170703,
+      "grad_norm": 17.25,
+      "kd_loss": 0.34608691930770874,
+      "learning_rate": 9.951590438446597e-06,
+      "loss": 0.2777,
+      "step": 220,
+      "student_loss": 0.06770455092191696,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06487889273356401,
+      "grad_norm": 37.0,
+      "kd_loss": 0.13642175495624542,
+      "learning_rate": 9.947771706172285e-06,
+      "loss": 0.2711,
+      "step": 225,
+      "student_loss": 0.029741473495960236,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06632064590542099,
+      "grad_norm": 13.5,
+      "kd_loss": 0.24233467876911163,
+      "learning_rate": 9.943808808055472e-06,
+      "loss": 0.2623,
+      "step": 230,
+      "student_loss": 0.0009022229351103306,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06776239907727798,
+      "grad_norm": 11.875,
+      "kd_loss": 0.6708654165267944,
+      "learning_rate": 9.93970185956522e-06,
+      "loss": 0.2862,
+      "step": 235,
+      "student_loss": 0.019873904064297676,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06920415224913495,
+      "grad_norm": 28.0,
+      "kd_loss": 0.18894419074058533,
+      "learning_rate": 9.93545098036786e-06,
+      "loss": 0.2625,
+      "step": 240,
+      "student_loss": 0.0006731516914442182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07064590542099193,
+      "grad_norm": 20.25,
+      "kd_loss": 0.13373719155788422,
+      "learning_rate": 9.931056294323507e-06,
+      "loss": 0.2638,
+      "step": 245,
+      "student_loss": 0.0005933287902735174,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07208765859284891,
+      "grad_norm": 10.0,
+      "kd_loss": 0.12812890112400055,
+      "learning_rate": 9.926517929482454e-06,
+      "loss": 0.2926,
+      "step": 250,
+      "student_loss": 0.00034213275648653507,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07352941176470588,
+      "grad_norm": 16.75,
+      "kd_loss": 0.08018118143081665,
+      "learning_rate": 9.921836018081438e-06,
+      "loss": 0.2428,
+      "step": 255,
+      "student_loss": 0.00025593984173610806,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07497116493656286,
+      "grad_norm": 10.75,
+      "kd_loss": 0.19167569279670715,
+      "learning_rate": 9.917010696539793e-06,
+      "loss": 0.305,
+      "step": 260,
+      "student_loss": 0.04356108978390694,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07641291810841984,
+      "grad_norm": 26.625,
+      "kd_loss": 0.0696549043059349,
+      "learning_rate": 9.912042105455462e-06,
+      "loss": 0.2747,
+      "step": 265,
+      "student_loss": 0.0003771705087274313,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07785467128027682,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.06586078554391861,
+      "learning_rate": 9.906930389600917e-06,
+      "loss": 0.2801,
+      "step": 270,
+      "student_loss": 0.00041506686829961836,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07929642445213379,
+      "grad_norm": 20.75,
+      "kd_loss": 0.14978329837322235,
+      "learning_rate": 9.901675697918925e-06,
+      "loss": 0.2886,
+      "step": 275,
+      "student_loss": 0.04765838012099266,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08073817762399077,
+      "grad_norm": 21.75,
+      "kd_loss": 0.17263588309288025,
+      "learning_rate": 9.896278183518216e-06,
+      "loss": 0.2897,
+      "step": 280,
+      "student_loss": 0.0004967491258867085,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08217993079584775,
+      "grad_norm": 20.0,
+      "kd_loss": 0.10580945014953613,
+      "learning_rate": 9.890738003669029e-06,
+      "loss": 0.293,
+      "step": 285,
+      "student_loss": 0.0005849753506481647,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08362168396770472,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.09176977723836899,
+      "learning_rate": 9.885055319798514e-06,
+      "loss": 0.3088,
+      "step": 290,
+      "student_loss": 0.0006238222122192383,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0850634371395617,
+      "grad_norm": 10.9375,
+      "kd_loss": 0.07891599833965302,
+      "learning_rate": 9.879230297486034e-06,
+      "loss": 0.2957,
+      "step": 295,
+      "student_loss": 0.0005840823869220912,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08650519031141868,
+      "grad_norm": 7.9375,
+      "kd_loss": 0.3616674840450287,
+      "learning_rate": 9.873263106458353e-06,
+      "loss": 0.2483,
+      "step": 300,
+      "student_loss": 0.019933484494686127,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08794694348327567,
+      "grad_norm": 15.9375,
+      "kd_loss": 0.15585829317569733,
+      "learning_rate": 9.86715392058467e-06,
+      "loss": 0.2522,
+      "step": 305,
+      "student_loss": 0.018561875447630882,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08938869665513265,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.06951466202735901,
+      "learning_rate": 9.860902917871566e-06,
+      "loss": 0.2646,
+      "step": 310,
+      "student_loss": 0.00046623480739071965,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09083044982698962,
+      "grad_norm": 11.5,
+      "kd_loss": 0.2836111783981323,
+      "learning_rate": 9.854510280457819e-06,
+      "loss": 0.2393,
+      "step": 315,
+      "student_loss": 0.012322928756475449,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0922722029988466,
+      "grad_norm": 13.6875,
+      "kd_loss": 0.10447105020284653,
+      "learning_rate": 9.847976194609083e-06,
+      "loss": 0.2668,
+      "step": 320,
+      "student_loss": 0.00048153125680983067,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09371395617070358,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.4263528883457184,
+      "learning_rate": 9.841300850712479e-06,
+      "loss": 0.2377,
+      "step": 325,
+      "student_loss": 0.00044430175330489874,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09515570934256055,
+      "grad_norm": 17.5,
+      "kd_loss": 0.6019083261489868,
+      "learning_rate": 9.834484443271035e-06,
+      "loss": 0.2697,
+      "step": 330,
+      "student_loss": 0.05738024041056633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09659746251441753,
+      "grad_norm": 7.125,
+      "kd_loss": 0.0715835765004158,
+      "learning_rate": 9.827527170898019e-06,
+      "loss": 0.2054,
+      "step": 335,
+      "student_loss": 0.0005960066919215024,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09803921568627451,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.10987883061170578,
+      "learning_rate": 9.820429236311158e-06,
+      "loss": 0.2695,
+      "step": 340,
+      "student_loss": 0.009573950432240963,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09948096885813149,
+      "grad_norm": 34.0,
+      "kd_loss": 0.17168864607810974,
+      "learning_rate": 9.81319084632673e-06,
+      "loss": 0.258,
+      "step": 345,
+      "student_loss": 0.016774671152234077,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10092272202998846,
+      "grad_norm": 21.375,
+      "kd_loss": 0.532006025314331,
+      "learning_rate": 9.805812211853534e-06,
+      "loss": 0.2368,
+      "step": 350,
+      "student_loss": 0.0042114779353141785,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10236447520184544,
+      "grad_norm": 16.625,
+      "kd_loss": 0.0480809211730957,
+      "learning_rate": 9.798293547886748e-06,
+      "loss": 0.2384,
+      "step": 355,
+      "student_loss": 0.000468328595161438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10380622837370242,
+      "grad_norm": 19.625,
+      "kd_loss": 0.06214166805148125,
+      "learning_rate": 9.790635073501662e-06,
+      "loss": 0.2473,
+      "step": 360,
+      "student_loss": 0.0004774518311023712,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1052479815455594,
+      "grad_norm": 11.0,
+      "kd_loss": 0.23296360671520233,
+      "learning_rate": 9.782837011847294e-06,
+      "loss": 0.3018,
+      "step": 365,
+      "student_loss": 0.045026302337646484,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10668973471741638,
+      "grad_norm": 14.5625,
+      "kd_loss": 0.39091649651527405,
+      "learning_rate": 9.774899590139897e-06,
+      "loss": 0.2487,
+      "step": 370,
+      "student_loss": 0.019704369828104973,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10813148788927336,
+      "grad_norm": 9.1875,
+      "kd_loss": 0.1680285483598709,
+      "learning_rate": 9.766823039656328e-06,
+      "loss": 0.2398,
+      "step": 375,
+      "student_loss": 0.038442861288785934,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10957324106113034,
+      "grad_norm": 17.875,
+      "kd_loss": 0.06734355539083481,
+      "learning_rate": 9.758607595727309e-06,
+      "loss": 0.2636,
+      "step": 380,
+      "student_loss": 0.0004427676321938634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11101499423298732,
+      "grad_norm": 16.25,
+      "kd_loss": 0.16070111095905304,
+      "learning_rate": 9.75025349773058e-06,
+      "loss": 0.2488,
+      "step": 385,
+      "student_loss": 0.0003361160634085536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11245674740484429,
+      "grad_norm": 10.0,
+      "kd_loss": 0.06088823080062866,
+      "learning_rate": 9.741760989083914e-06,
+      "loss": 0.2351,
+      "step": 390,
+      "student_loss": 0.00025634359917603433,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11389850057670127,
+      "grad_norm": 17.375,
+      "kd_loss": 0.1013672724366188,
+      "learning_rate": 9.73313031723803e-06,
+      "loss": 0.2348,
+      "step": 395,
+      "student_loss": 0.00046920194290578365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11534025374855825,
+      "grad_norm": 12.9375,
+      "kd_loss": 0.12313513457775116,
+      "learning_rate": 9.724361733669383e-06,
+      "loss": 0.1916,
+      "step": 400,
+      "student_loss": 0.0006792352651245892,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11678200692041522,
+      "grad_norm": 12.4375,
+      "kd_loss": 0.11874714493751526,
+      "learning_rate": 9.715455493872827e-06,
+      "loss": 0.2445,
+      "step": 405,
+      "student_loss": 0.0060541993007063866,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1182237600922722,
+      "grad_norm": 15.875,
+      "kd_loss": 0.3652341663837433,
+      "learning_rate": 9.70641185735419e-06,
+      "loss": 0.2546,
+      "step": 410,
+      "student_loss": 0.017243457958102226,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11966551326412918,
+      "grad_norm": 14.0,
+      "kd_loss": 0.1578780561685562,
+      "learning_rate": 9.697231087622691e-06,
+      "loss": 0.2759,
+      "step": 415,
+      "student_loss": 0.0010528146522119641,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12110726643598616,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.3285977840423584,
+      "learning_rate": 9.687913452183277e-06,
+      "loss": 0.2341,
+      "step": 420,
+      "student_loss": 0.06141834706068039,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12254901960784313,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.05784276872873306,
+      "learning_rate": 9.678459222528825e-06,
+      "loss": 0.2234,
+      "step": 425,
+      "student_loss": 0.0011342725483700633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12399077277970011,
+      "grad_norm": 11.8125,
+      "kd_loss": 0.35227036476135254,
+      "learning_rate": 9.668868674132224e-06,
+      "loss": 0.3045,
+      "step": 430,
+      "student_loss": 0.05136450007557869,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1254325259515571,
+      "grad_norm": 13.0625,
+      "kd_loss": 0.09926026314496994,
+      "learning_rate": 9.65914208643836e-06,
+      "loss": 0.2054,
+      "step": 435,
+      "student_loss": 0.0007687933393754065,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12687427912341406,
+      "grad_norm": 10.8125,
+      "kd_loss": 0.3320644795894623,
+      "learning_rate": 9.649279742855964e-06,
+      "loss": 0.2635,
+      "step": 440,
+      "student_loss": 0.02549891546368599,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12831603229527105,
+      "grad_norm": 10.0,
+      "kd_loss": 0.19906648993492126,
+      "learning_rate": 9.639281930749363e-06,
+      "loss": 0.2341,
+      "step": 445,
+      "student_loss": 0.025979263707995415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12975778546712802,
+      "grad_norm": 9.25,
+      "kd_loss": 0.164601668715477,
+      "learning_rate": 9.629148941430096e-06,
+      "loss": 0.2751,
+      "step": 450,
+      "student_loss": 0.0007244550506584346,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.131199538638985,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.04286447539925575,
+      "learning_rate": 9.618881070148436e-06,
+      "loss": 0.2417,
+      "step": 455,
+      "student_loss": 0.000796566077042371,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13264129181084197,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.17657798528671265,
+      "learning_rate": 9.608478616084784e-06,
+      "loss": 0.1985,
+      "step": 460,
+      "student_loss": 0.00047529107541777194,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13408304498269896,
+      "grad_norm": 6.90625,
+      "kd_loss": 0.2981259524822235,
+      "learning_rate": 9.597941882340947e-06,
+      "loss": 0.2786,
+      "step": 465,
+      "student_loss": 0.00040858631837181747,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13552479815455595,
+      "grad_norm": 25.0,
+      "kd_loss": 0.09742365032434464,
+      "learning_rate": 9.587271175931313e-06,
+      "loss": 0.2365,
+      "step": 470,
+      "student_loss": 0.0003401920839678496,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13696655132641292,
+      "grad_norm": 23.625,
+      "kd_loss": 0.24699804186820984,
+      "learning_rate": 9.5764668077739e-06,
+      "loss": 0.1986,
+      "step": 475,
+      "student_loss": 0.0005240993923507631,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1384083044982699,
+      "grad_norm": 21.75,
+      "kd_loss": 0.19472193717956543,
+      "learning_rate": 9.5655290926813e-06,
+      "loss": 0.2274,
+      "step": 480,
+      "student_loss": 0.008671707473695278,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13985005767012687,
+      "grad_norm": 17.125,
+      "kd_loss": 0.055427636951208115,
+      "learning_rate": 9.554458349351507e-06,
+      "loss": 0.2293,
+      "step": 485,
+      "student_loss": 0.0014853382017463446,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14129181084198386,
+      "grad_norm": 13.3125,
+      "kd_loss": 0.2377857267856598,
+      "learning_rate": 9.54325490035863e-06,
+      "loss": 0.2595,
+      "step": 490,
+      "student_loss": 0.000622899504378438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14273356401384082,
+      "grad_norm": 9.3125,
+      "kd_loss": 0.3495025932788849,
+      "learning_rate": 9.531919072143488e-06,
+      "loss": 0.2468,
+      "step": 495,
+      "student_loss": 0.0005211852840147913,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14417531718569782,
+      "grad_norm": 10.3125,
+      "kd_loss": 0.18743589520454407,
+      "learning_rate": 9.520451195004107e-06,
+      "loss": 0.2461,
+      "step": 500,
+      "student_loss": 0.0002995518152602017,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14561707035755478,
+      "grad_norm": 12.5,
+      "kd_loss": 0.23420903086662292,
+      "learning_rate": 9.508851603086094e-06,
+      "loss": 0.2539,
+      "step": 505,
+      "student_loss": 0.0004655766242649406,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14705882352941177,
+      "grad_norm": 20.375,
+      "kd_loss": 0.06832753121852875,
+      "learning_rate": 9.497120634372894e-06,
+      "loss": 0.261,
+      "step": 510,
+      "student_loss": 0.0003136875166092068,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14850057670126873,
+      "grad_norm": 8.9375,
+      "kd_loss": 0.09611348807811737,
+      "learning_rate": 9.48525863067595e-06,
+      "loss": 0.2116,
+      "step": 515,
+      "student_loss": 0.007644087541848421,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14994232987312572,
+      "grad_norm": 7.28125,
+      "kd_loss": 0.2377752661705017,
+      "learning_rate": 9.473265937624748e-06,
+      "loss": 0.217,
+      "step": 520,
+      "student_loss": 0.04185197129845619,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1513840830449827,
+      "grad_norm": 23.375,
+      "kd_loss": 0.20080943405628204,
+      "learning_rate": 9.461142904656722e-06,
+      "loss": 0.249,
+      "step": 525,
+      "student_loss": 0.000431343651143834,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15282583621683968,
+      "grad_norm": 13.5625,
+      "kd_loss": 0.0958016961812973,
+      "learning_rate": 9.448889885007105e-06,
+      "loss": 0.2305,
+      "step": 530,
+      "student_loss": 0.0003980971814598888,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15426758938869667,
+      "grad_norm": 21.625,
+      "kd_loss": 0.37275564670562744,
+      "learning_rate": 9.436507235698613e-06,
+      "loss": 0.2309,
+      "step": 535,
+      "student_loss": 0.0004774174012709409,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15570934256055363,
+      "grad_norm": 31.375,
+      "kd_loss": 0.17777006328105927,
+      "learning_rate": 9.423995317531049e-06,
+      "loss": 0.2529,
+      "step": 540,
+      "student_loss": 0.0005782285006716847,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15715109573241062,
+      "grad_norm": 17.125,
+      "kd_loss": 0.07950557768344879,
+      "learning_rate": 9.4113544950708e-06,
+      "loss": 0.2076,
+      "step": 545,
+      "student_loss": 0.00357666308991611,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15859284890426759,
+      "grad_norm": 17.0,
+      "kd_loss": 0.15328699350357056,
+      "learning_rate": 9.398585136640195e-06,
+      "loss": 0.2046,
+      "step": 550,
+      "student_loss": 0.0001933475286932662,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16003460207612458,
+      "grad_norm": 20.25,
+      "kd_loss": 0.5593510866165161,
+      "learning_rate": 9.38568761430679e-06,
+      "loss": 0.2422,
+      "step": 555,
+      "student_loss": 0.06740490347146988,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16147635524798154,
+      "grad_norm": 16.875,
+      "kd_loss": 0.4548700153827667,
+      "learning_rate": 9.372662303872514e-06,
+      "loss": 0.2419,
+      "step": 560,
+      "student_loss": 0.0748598724603653,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16291810841983853,
+      "grad_norm": 9.375,
+      "kd_loss": 0.22124871611595154,
+      "learning_rate": 9.359509584862735e-06,
+      "loss": 0.2165,
+      "step": 565,
+      "student_loss": 0.03530263155698776,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1643598615916955,
+      "grad_norm": 17.875,
+      "kd_loss": 0.26488637924194336,
+      "learning_rate": 9.346229840515186e-06,
+      "loss": 0.2562,
+      "step": 570,
+      "student_loss": 0.010391860269010067,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16580161476355249,
+      "grad_norm": 15.3125,
+      "kd_loss": 0.09221018850803375,
+      "learning_rate": 9.332823457768805e-06,
+      "loss": 0.2131,
+      "step": 575,
+      "student_loss": 0.0004648648900911212,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16724336793540945,
+      "grad_norm": 9.625,
+      "kd_loss": 0.153616264462471,
+      "learning_rate": 9.31929082725246e-06,
+      "loss": 0.205,
+      "step": 580,
+      "student_loss": 0.0011936463415622711,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16868512110726644,
+      "grad_norm": 37.75,
+      "kd_loss": 0.15198436379432678,
+      "learning_rate": 9.305632343273567e-06,
+      "loss": 0.1994,
+      "step": 585,
+      "student_loss": 0.0003124019131064415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1701268742791234,
+      "grad_norm": 9.75,
+      "kd_loss": 0.17398428916931152,
+      "learning_rate": 9.291848403806604e-06,
+      "loss": 0.2032,
+      "step": 590,
+      "student_loss": 0.0004771619278471917,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1715686274509804,
+      "grad_norm": 17.25,
+      "kd_loss": 0.10845454782247543,
+      "learning_rate": 9.277939410481507e-06,
+      "loss": 0.2295,
+      "step": 595,
+      "student_loss": 0.0003602717479225248,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17301038062283736,
+      "grad_norm": 21.75,
+      "kd_loss": 0.2581179738044739,
+      "learning_rate": 9.263905768571977e-06,
+      "loss": 0.2434,
+      "step": 600,
+      "student_loss": 0.04208357632160187,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17445213379469435,
+      "grad_norm": 20.75,
+      "kd_loss": 0.19223031401634216,
+      "learning_rate": 9.249747886983664e-06,
+      "loss": 0.2328,
+      "step": 605,
+      "student_loss": 0.07404639571905136,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17589388696655134,
+      "grad_norm": 13.5625,
+      "kd_loss": 0.3590433597564697,
+      "learning_rate": 9.235466178242255e-06,
+      "loss": 0.238,
+      "step": 610,
+      "student_loss": 0.0011161614675074816,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1773356401384083,
+      "grad_norm": 26.625,
+      "kd_loss": 0.06018394976854324,
+      "learning_rate": 9.221061058481456e-06,
+      "loss": 0.1984,
+      "step": 615,
+      "student_loss": 0.0002010497555602342,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1787773933102653,
+      "grad_norm": 16.5,
+      "kd_loss": 0.2592211663722992,
+      "learning_rate": 9.206532947430865e-06,
+      "loss": 0.2219,
+      "step": 620,
+      "student_loss": 0.013502068817615509,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18021914648212226,
+      "grad_norm": 16.0,
+      "kd_loss": 0.09846622496843338,
+      "learning_rate": 9.191882268403743e-06,
+      "loss": 0.2017,
+      "step": 625,
+      "student_loss": 0.0058509972877800465,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18166089965397925,
+      "grad_norm": 27.75,
+      "kd_loss": 0.5237650871276855,
+      "learning_rate": 9.177109448284673e-06,
+      "loss": 0.236,
+      "step": 630,
+      "student_loss": 0.06524383276700974,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1831026528258362,
+      "grad_norm": 18.0,
+      "kd_loss": 0.13129356503486633,
+      "learning_rate": 9.162214917517139e-06,
+      "loss": 0.2477,
+      "step": 635,
+      "student_loss": 0.00046817411202937365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1845444059976932,
+      "grad_norm": 12.125,
+      "kd_loss": 0.09964244067668915,
+      "learning_rate": 9.14719911009096e-06,
+      "loss": 0.1964,
+      "step": 640,
+      "student_loss": 0.00038446439430117607,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18598615916955016,
+      "grad_norm": 12.3125,
+      "kd_loss": 0.1809924840927124,
+      "learning_rate": 9.132062463529666e-06,
+      "loss": 0.2034,
+      "step": 645,
+      "student_loss": 0.0003987185191363096,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18742791234140715,
+      "grad_norm": 43.75,
+      "kd_loss": 0.07568997144699097,
+      "learning_rate": 9.11680541887774e-06,
+      "loss": 0.2124,
+      "step": 650,
+      "student_loss": 0.0005847765132784843,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18886966551326412,
+      "grad_norm": 32.0,
+      "kd_loss": 0.18582190573215485,
+      "learning_rate": 9.101428420687759e-06,
+      "loss": 0.2401,
+      "step": 655,
+      "student_loss": 0.00038169475737959146,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1903114186851211,
+      "grad_norm": 28.75,
+      "kd_loss": 0.1885007619857788,
+      "learning_rate": 9.085931917007464e-06,
+      "loss": 0.2095,
+      "step": 660,
+      "student_loss": 0.0004360108287073672,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19175317185697807,
+      "grad_norm": 16.625,
+      "kd_loss": 0.4099441170692444,
+      "learning_rate": 9.070316359366681e-06,
+      "loss": 0.2286,
+      "step": 665,
+      "student_loss": 0.0005011683679185808,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19319492502883506,
+      "grad_norm": 13.75,
+      "kd_loss": 0.2379698008298874,
+      "learning_rate": 9.054582202764175e-06,
+      "loss": 0.202,
+      "step": 670,
+      "student_loss": 0.0059415721334517,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19463667820069205,
+      "grad_norm": 7.5,
+      "kd_loss": 0.05046012997627258,
+      "learning_rate": 9.038729905654393e-06,
+      "loss": 0.2227,
+      "step": 675,
+      "student_loss": 0.00044423757935874164,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19607843137254902,
+      "grad_norm": 14.125,
+      "kd_loss": 0.2490309476852417,
+      "learning_rate": 9.022759929934109e-06,
+      "loss": 0.1973,
+      "step": 680,
+      "student_loss": 0.07809463143348694,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.197520184544406,
+      "grad_norm": 19.25,
+      "kd_loss": 0.24093234539031982,
+      "learning_rate": 9.006672740928952e-06,
+      "loss": 0.2393,
+      "step": 685,
+      "student_loss": 0.044076789170503616,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19896193771626297,
+      "grad_norm": 19.25,
+      "kd_loss": 0.7235217094421387,
+      "learning_rate": 8.990468807379861e-06,
+      "loss": 0.2368,
+      "step": 690,
+      "student_loss": 0.054187677800655365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20040369088811996,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.11449181288480759,
+      "learning_rate": 8.974148601429426e-06,
+      "loss": 0.2614,
+      "step": 695,
+      "student_loss": 0.0049613662995398045,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20184544405997693,
+      "grad_norm": 10.625,
+      "kd_loss": 0.06947001069784164,
+      "learning_rate": 8.957712598608123e-06,
+      "loss": 0.2167,
+      "step": 700,
+      "student_loss": 0.0008642565226182342,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20328719723183392,
+      "grad_norm": 12.5625,
+      "kd_loss": 0.2155514508485794,
+      "learning_rate": 8.941161277820462e-06,
+      "loss": 0.2222,
+      "step": 705,
+      "student_loss": 0.0004866976523771882,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20472895040369088,
+      "grad_norm": 11.25,
+      "kd_loss": 0.45056015253067017,
+      "learning_rate": 8.924495121331038e-06,
+      "loss": 0.2276,
+      "step": 710,
+      "student_loss": 0.008678597398102283,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20617070357554787,
+      "grad_norm": 23.875,
+      "kd_loss": 0.08582590520381927,
+      "learning_rate": 8.907714614750473e-06,
+      "loss": 0.2311,
+      "step": 715,
+      "student_loss": 0.01213450264185667,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20761245674740483,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.16561608016490936,
+      "learning_rate": 8.890820247021263e-06,
+      "loss": 0.23,
+      "step": 720,
+      "student_loss": 0.002681850688531995,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20905420991926182,
+      "grad_norm": 20.875,
+      "kd_loss": 0.07642826437950134,
+      "learning_rate": 8.873812510403545e-06,
+      "loss": 0.1851,
+      "step": 725,
+      "student_loss": 0.00026688360958360136,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2104959630911188,
+      "grad_norm": 17.25,
+      "kd_loss": 0.17072460055351257,
+      "learning_rate": 8.85669190046074e-06,
+      "loss": 0.2456,
+      "step": 730,
+      "student_loss": 0.017637869343161583,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21193771626297578,
+      "grad_norm": 13.25,
+      "kd_loss": 0.1058090403676033,
+      "learning_rate": 8.839458916045117e-06,
+      "loss": 0.2502,
+      "step": 735,
+      "student_loss": 0.0003498170990496874,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21337946943483277,
+      "grad_norm": 6.5,
+      "kd_loss": 0.2888850271701813,
+      "learning_rate": 8.822114059283267e-06,
+      "loss": 0.2033,
+      "step": 740,
+      "student_loss": 0.002009955933317542,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21482122260668973,
+      "grad_norm": 7.625,
+      "kd_loss": 0.11317810416221619,
+      "learning_rate": 8.804657835561456e-06,
+      "loss": 0.2178,
+      "step": 745,
+      "student_loss": 0.04010392725467682,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21626297577854672,
+      "grad_norm": 17.5,
+      "kd_loss": 0.7453857064247131,
+      "learning_rate": 8.787090753510917e-06,
+      "loss": 0.1926,
+      "step": 750,
+      "student_loss": 0.024328170344233513,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2177047289504037,
+      "grad_norm": 10.6875,
+      "kd_loss": 0.06280636787414551,
+      "learning_rate": 8.76941332499301e-06,
+      "loss": 0.2399,
+      "step": 755,
+      "student_loss": 0.032401587814092636,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21914648212226068,
+      "grad_norm": 11.6875,
+      "kd_loss": 0.17805851995944977,
+      "learning_rate": 8.751626065084328e-06,
+      "loss": 0.2135,
+      "step": 760,
+      "student_loss": 0.022408630698919296,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22058823529411764,
+      "grad_norm": 10.0625,
+      "kd_loss": 0.3160823881626129,
+      "learning_rate": 8.733729492061675e-06,
+      "loss": 0.206,
+      "step": 765,
+      "student_loss": 0.005015719681978226,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22202998846597463,
+      "grad_norm": 9.9375,
+      "kd_loss": 0.31819650530815125,
+      "learning_rate": 8.715724127386971e-06,
+      "loss": 0.2282,
+      "step": 770,
+      "student_loss": 0.0012502025347203016,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2234717416378316,
+      "grad_norm": 20.25,
+      "kd_loss": 0.31151849031448364,
+      "learning_rate": 8.697610495692055e-06,
+      "loss": 0.1895,
+      "step": 775,
+      "student_loss": 0.0014239229494705796,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22491349480968859,
+      "grad_norm": 13.0625,
+      "kd_loss": 0.2508423626422882,
+      "learning_rate": 8.679389124763396e-06,
+      "loss": 0.2453,
+      "step": 780,
+      "student_loss": 0.0013981886440888047,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22635524798154555,
+      "grad_norm": 20.875,
+      "kd_loss": 0.04912475869059563,
+      "learning_rate": 8.661060545526724e-06,
+      "loss": 0.2078,
+      "step": 785,
+      "student_loss": 0.00030144155607558787,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22779700115340254,
+      "grad_norm": 18.25,
+      "kd_loss": 0.05954095348715782,
+      "learning_rate": 8.64262529203155e-06,
+      "loss": 0.1789,
+      "step": 790,
+      "student_loss": 0.001504220417700708,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2292387543252595,
+      "grad_norm": 8.1875,
+      "kd_loss": 0.1065668910741806,
+      "learning_rate": 8.624083901435613e-06,
+      "loss": 0.2176,
+      "step": 795,
+      "student_loss": 0.00033886852907016873,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2306805074971165,
+      "grad_norm": 16.125,
+      "kd_loss": 0.5510667562484741,
+      "learning_rate": 8.60543691398922e-06,
+      "loss": 0.2246,
+      "step": 800,
+      "student_loss": 0.06318172067403793,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23212226066897348,
+      "grad_norm": 12.5,
+      "kd_loss": 0.0667741522192955,
+      "learning_rate": 8.586684873019513e-06,
+      "loss": 0.2272,
+      "step": 805,
+      "student_loss": 0.0003553811402525753,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23356401384083045,
+      "grad_norm": 8.875,
+      "kd_loss": 0.12136233597993851,
+      "learning_rate": 8.567828324914633e-06,
+      "loss": 0.1892,
+      "step": 810,
+      "student_loss": 0.0004278223786968738,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23500576701268744,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.11104825884103775,
+      "learning_rate": 8.548867819107803e-06,
+      "loss": 0.2218,
+      "step": 815,
+      "student_loss": 0.0006955342250876129,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2364475201845444,
+      "grad_norm": 8.625,
+      "kd_loss": 0.21142087876796722,
+      "learning_rate": 8.52980390806131e-06,
+      "loss": 0.1921,
+      "step": 820,
+      "student_loss": 0.007228876929730177,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2378892733564014,
+      "grad_norm": 10.875,
+      "kd_loss": 0.17360462248325348,
+      "learning_rate": 8.51063714725042e-06,
+      "loss": 0.2136,
+      "step": 825,
+      "student_loss": 0.0012437300756573677,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23933102652825836,
+      "grad_norm": 9.625,
+      "kd_loss": 0.3183634877204895,
+      "learning_rate": 8.49136809514719e-06,
+      "loss": 0.2384,
+      "step": 830,
+      "student_loss": 0.004646586254239082,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24077277970011535,
+      "grad_norm": 17.25,
+      "kd_loss": 0.17037151753902435,
+      "learning_rate": 8.471997313204183e-06,
+      "loss": 0.1873,
+      "step": 835,
+      "student_loss": 0.015499490313231945,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2422145328719723,
+      "grad_norm": 18.0,
+      "kd_loss": 0.14555610716342926,
+      "learning_rate": 8.452525365838123e-06,
+      "loss": 0.2059,
+      "step": 840,
+      "student_loss": 0.0005742039065808058,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2436562860438293,
+      "grad_norm": 15.8125,
+      "kd_loss": 0.06973056495189667,
+      "learning_rate": 8.432952820413447e-06,
+      "loss": 0.2197,
+      "step": 845,
+      "student_loss": 0.0005098649999126792,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24509803921568626,
+      "grad_norm": 10.625,
+      "kd_loss": 0.2858434021472931,
+      "learning_rate": 8.41328024722577e-06,
+      "loss": 0.2115,
+      "step": 850,
+      "student_loss": 0.012342527508735657,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24653979238754326,
+      "grad_norm": 15.0625,
+      "kd_loss": 0.09209071844816208,
+      "learning_rate": 8.393508219485263e-06,
+      "loss": 0.2095,
+      "step": 855,
+      "student_loss": 0.00024691459839232266,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24798154555940022,
+      "grad_norm": 8.625,
+      "kd_loss": 0.08900707215070724,
+      "learning_rate": 8.373637313299966e-06,
+      "loss": 0.2341,
+      "step": 860,
+      "student_loss": 0.0005091276834718883,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2494232987312572,
+      "grad_norm": 18.125,
+      "kd_loss": 0.11508733034133911,
+      "learning_rate": 8.353668107658984e-06,
+      "loss": 0.2099,
+      "step": 865,
+      "student_loss": 0.0002668049419298768,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2508650519031142,
+      "grad_norm": 15.4375,
+      "kd_loss": 0.29352444410324097,
+      "learning_rate": 8.333601184415632e-06,
+      "loss": 0.2173,
+      "step": 870,
+      "student_loss": 0.0016936544561758637,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25230680507497116,
+      "grad_norm": 9.25,
+      "kd_loss": 0.05814331769943237,
+      "learning_rate": 8.31343712827047e-06,
+      "loss": 0.2007,
+      "step": 875,
+      "student_loss": 0.00016406318172812462,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2537485582468281,
+      "grad_norm": 8.0625,
+      "kd_loss": 0.12616939842700958,
+      "learning_rate": 8.293176526754274e-06,
+      "loss": 0.2079,
+      "step": 880,
+      "student_loss": 0.00044341504690237343,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25519031141868515,
+      "grad_norm": 9.875,
+      "kd_loss": 0.6110478639602661,
+      "learning_rate": 8.27281997021091e-06,
+      "loss": 0.1749,
+      "step": 885,
+      "student_loss": 0.03696923702955246,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2566320645905421,
+      "grad_norm": 12.875,
+      "kd_loss": 0.049601469188928604,
+      "learning_rate": 8.25236805178014e-06,
+      "loss": 0.2115,
+      "step": 890,
+      "student_loss": 0.00036779773654416203,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25807381776239907,
+      "grad_norm": 7.46875,
+      "kd_loss": 0.3071838617324829,
+      "learning_rate": 8.231821367380335e-06,
+      "loss": 0.2182,
+      "step": 895,
+      "student_loss": 0.0032658062409609556,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25951557093425603,
+      "grad_norm": 19.25,
+      "kd_loss": 0.1950993835926056,
+      "learning_rate": 8.211180515691108e-06,
+      "loss": 0.2058,
+      "step": 900,
+      "student_loss": 0.03542155399918556,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26095732410611305,
+      "grad_norm": 12.125,
+      "kd_loss": 0.4699344038963318,
+      "learning_rate": 8.190446098135883e-06,
+      "loss": 0.1775,
+      "step": 905,
+      "student_loss": 0.04188324883580208,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26239907727797,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.04896146059036255,
+      "learning_rate": 8.16961871886435e-06,
+      "loss": 0.1931,
+      "step": 910,
+      "student_loss": 0.00029122710111550987,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.263840830449827,
+      "grad_norm": 23.5,
+      "kd_loss": 0.514575183391571,
+      "learning_rate": 8.148698984734887e-06,
+      "loss": 0.1831,
+      "step": 915,
+      "student_loss": 0.09658902883529663,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26528258362168394,
+      "grad_norm": 20.125,
+      "kd_loss": 0.16474592685699463,
+      "learning_rate": 8.127687505296852e-06,
+      "loss": 0.2031,
+      "step": 920,
+      "student_loss": 0.020180951803922653,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26672433679354096,
+      "grad_norm": 13.6875,
+      "kd_loss": 0.03160429745912552,
+      "learning_rate": 8.106584892772844e-06,
+      "loss": 0.1953,
+      "step": 925,
+      "student_loss": 0.0002911738120019436,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2681660899653979,
+      "grad_norm": 9.875,
+      "kd_loss": 0.09231647849082947,
+      "learning_rate": 8.08539176204085e-06,
+      "loss": 0.1953,
+      "step": 930,
+      "student_loss": 0.02371821738779545,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2696078431372549,
+      "grad_norm": 23.875,
+      "kd_loss": 0.11771639436483383,
+      "learning_rate": 8.064108730616336e-06,
+      "loss": 0.1856,
+      "step": 935,
+      "student_loss": 0.0008747431566007435,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2710495963091119,
+      "grad_norm": 14.9375,
+      "kd_loss": 0.24315571784973145,
+      "learning_rate": 8.042736418634252e-06,
+      "loss": 0.2151,
+      "step": 940,
+      "student_loss": 0.011635007336735725,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27249134948096887,
+      "grad_norm": 14.1875,
+      "kd_loss": 0.08452239632606506,
+      "learning_rate": 8.02127544883096e-06,
+      "loss": 0.2101,
+      "step": 945,
+      "student_loss": 0.0033563401084393263,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27393310265282583,
+      "grad_norm": 19.5,
+      "kd_loss": 0.12136533856391907,
+      "learning_rate": 7.999726446526095e-06,
+      "loss": 0.1898,
+      "step": 950,
+      "student_loss": 0.0002544254530221224,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2753748558246828,
+      "grad_norm": 18.25,
+      "kd_loss": 0.12915752828121185,
+      "learning_rate": 7.978090039604342e-06,
+      "loss": 0.1979,
+      "step": 955,
+      "student_loss": 0.0007313934620469809,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2768166089965398,
+      "grad_norm": 10.375,
+      "kd_loss": 0.16751284897327423,
+      "learning_rate": 7.956366858497138e-06,
+      "loss": 0.1886,
+      "step": 960,
+      "student_loss": 0.0003871220105793327,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2782583621683968,
+      "grad_norm": 9.6875,
+      "kd_loss": 0.25768253207206726,
+      "learning_rate": 7.934557536164304e-06,
+      "loss": 0.1998,
+      "step": 965,
+      "student_loss": 0.013239127583801746,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27970011534025374,
+      "grad_norm": 25.5,
+      "kd_loss": 0.11466177552938461,
+      "learning_rate": 7.91266270807561e-06,
+      "loss": 0.202,
+      "step": 970,
+      "student_loss": 0.0003071602841373533,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2811418685121107,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.08647243678569794,
+      "learning_rate": 7.890683012192249e-06,
+      "loss": 0.2151,
+      "step": 975,
+      "student_loss": 0.017678463831543922,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2825836216839677,
+      "grad_norm": 11.0,
+      "kd_loss": 0.09852449595928192,
+      "learning_rate": 7.868619088948246e-06,
+      "loss": 0.191,
+      "step": 980,
+      "student_loss": 0.00038518354995176196,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2840253748558247,
+      "grad_norm": 11.25,
+      "kd_loss": 0.09779661893844604,
+      "learning_rate": 7.846471581231814e-06,
+      "loss": 0.2316,
+      "step": 985,
+      "student_loss": 0.00031637900974601507,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.28546712802768165,
+      "grad_norm": 14.0,
+      "kd_loss": 0.08365006744861603,
+      "learning_rate": 7.824241134366604e-06,
+      "loss": 0.1693,
+      "step": 990,
+      "student_loss": 0.00032978563103824854,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2869088811995386,
+      "grad_norm": 20.0,
+      "kd_loss": 0.29525840282440186,
+      "learning_rate": 7.801928396092908e-06,
+      "loss": 0.2244,
+      "step": 995,
+      "student_loss": 0.00505430344492197,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.28835063437139563,
+      "grad_norm": 7.53125,
+      "kd_loss": 0.1885785460472107,
+      "learning_rate": 7.779534016548791e-06,
+      "loss": 0.2048,
+      "step": 1000,
+      "student_loss": 0.0076581151224672794,
+      "teacher_loss": 0.0
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9f8166a8666359a7d79f910bbd0ebd8cad87f12a11e0d86c518e7e9694ca2ee
+size 3096275515

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/thought_projector.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:630007506a71597b85da147e55e17fe2875e5367cc6b7bfc474c9406bc3e33c0
+size 9445953

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-1500/trainer_state.json

@@ -0,0 +1,3034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.43252595155709345,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014417531718569781,
+      "grad_norm": 322.0,
+      "kd_loss": 1.997420072555542,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 2.1422,
+      "step": 5,
+      "student_loss": 0.0605761781334877,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0028835063437139563,
+      "grad_norm": 186.0,
+      "kd_loss": 1.8595829010009766,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 2.0829,
+      "step": 10,
+      "student_loss": 0.09249057620763779,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.004325259515570935,
+      "grad_norm": 117.5,
+      "kd_loss": 1.5637085437774658,
+      "learning_rate": 1.5555555555555558e-06,
+      "loss": 1.972,
+      "step": 15,
+      "student_loss": 0.026328546926379204,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0057670126874279125,
+      "grad_norm": 106.5,
+      "kd_loss": 1.4912058115005493,
+      "learning_rate": 2.1111111111111114e-06,
+      "loss": 1.8011,
+      "step": 20,
+      "student_loss": 0.13309946656227112,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00720876585928489,
+      "grad_norm": 86.5,
+      "kd_loss": 1.1542646884918213,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 1.5427,
+      "step": 25,
+      "student_loss": 0.036511313170194626,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00865051903114187,
+      "grad_norm": 58.5,
+      "kd_loss": 0.9403514266014099,
+      "learning_rate": 3.2222222222222227e-06,
+      "loss": 1.2794,
+      "step": 30,
+      "student_loss": 0.01801167242228985,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.010092272202998846,
+      "grad_norm": 31.5,
+      "kd_loss": 0.6764306426048279,
+      "learning_rate": 3.777777777777778e-06,
+      "loss": 1.0897,
+      "step": 35,
+      "student_loss": 0.0011638500727713108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.011534025374855825,
+      "grad_norm": 27.625,
+      "kd_loss": 0.7702282667160034,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 0.9024,
+      "step": 40,
+      "student_loss": 0.08332191407680511,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.012975778546712802,
+      "grad_norm": 20.875,
+      "kd_loss": 1.050789475440979,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.7206,
+      "step": 45,
+      "student_loss": 0.07726961374282837,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01441753171856978,
+      "grad_norm": 44.25,
+      "kd_loss": 0.4379478096961975,
+      "learning_rate": 5.444444444444445e-06,
+      "loss": 0.6754,
+      "step": 50,
+      "student_loss": 0.007436605170369148,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.015859284890426758,
+      "grad_norm": 17.5,
+      "kd_loss": 0.3016382157802582,
+      "learning_rate": 6e-06,
+      "loss": 0.5857,
+      "step": 55,
+      "student_loss": 0.002162981079891324,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01730103806228374,
+      "grad_norm": 24.625,
+      "kd_loss": 0.2297545075416565,
+      "learning_rate": 6.555555555555556e-06,
+      "loss": 0.5265,
+      "step": 60,
+      "student_loss": 0.012505847960710526,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.018742791234140715,
+      "grad_norm": 14.75,
+      "kd_loss": 0.2053602635860443,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 0.5101,
+      "step": 65,
+      "student_loss": 0.04744679108262062,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.020184544405997693,
+      "grad_norm": 34.0,
+      "kd_loss": 0.7437167763710022,
+      "learning_rate": 7.666666666666667e-06,
+      "loss": 0.4523,
+      "step": 70,
+      "student_loss": 0.029973715543746948,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02162629757785467,
+      "grad_norm": 14.0625,
+      "kd_loss": 0.18727970123291016,
+      "learning_rate": 8.222222222222222e-06,
+      "loss": 0.4534,
+      "step": 75,
+      "student_loss": 0.001686512609012425,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02306805074971165,
+      "grad_norm": 31.0,
+      "kd_loss": 0.18602637946605682,
+      "learning_rate": 8.777777777777778e-06,
+      "loss": 0.4338,
+      "step": 80,
+      "student_loss": 0.0014192159287631512,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.024509803921568627,
+      "grad_norm": 14.75,
+      "kd_loss": 0.11737873405218124,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.4378,
+      "step": 85,
+      "student_loss": 0.000477277091704309,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.025951557093425604,
+      "grad_norm": 17.125,
+      "kd_loss": 0.7024360299110413,
+      "learning_rate": 9.88888888888889e-06,
+      "loss": 0.3816,
+      "step": 90,
+      "student_loss": 0.027783483266830444,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.027393310265282585,
+      "grad_norm": 19.125,
+      "kd_loss": 0.3916384279727936,
+      "learning_rate": 9.99995337991255e-06,
+      "loss": 0.3905,
+      "step": 95,
+      "student_loss": 0.013136160559952259,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02883506343713956,
+      "grad_norm": 13.9375,
+      "kd_loss": 0.09685378521680832,
+      "learning_rate": 9.999763987297266e-06,
+      "loss": 0.3485,
+      "step": 100,
+      "student_loss": 0.0006464557372964919,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03027681660899654,
+      "grad_norm": 12.5,
+      "kd_loss": 0.48202934861183167,
+      "learning_rate": 9.999428913912876e-06,
+      "loss": 0.3555,
+      "step": 105,
+      "student_loss": 0.005315156653523445,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.031718569780853516,
+      "grad_norm": 9.8125,
+      "kd_loss": 0.42474475502967834,
+      "learning_rate": 9.998948169522587e-06,
+      "loss": 0.3616,
+      "step": 110,
+      "student_loss": 0.0304458886384964,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03316032295271049,
+      "grad_norm": 11.9375,
+      "kd_loss": 0.30219101905822754,
+      "learning_rate": 9.998321768134101e-06,
+      "loss": 0.3669,
+      "step": 115,
+      "student_loss": 0.0007903591031208634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03460207612456748,
+      "grad_norm": 47.0,
+      "kd_loss": 0.14825306832790375,
+      "learning_rate": 9.997549727999212e-06,
+      "loss": 0.3014,
+      "step": 120,
+      "student_loss": 0.0007933085435070097,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.036043829296424454,
+      "grad_norm": 10.6875,
+      "kd_loss": 0.6828317046165466,
+      "learning_rate": 9.996632071613257e-06,
+      "loss": 0.3416,
+      "step": 125,
+      "student_loss": 0.056072596460580826,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03748558246828143,
+      "grad_norm": 21.5,
+      "kd_loss": 0.16518603265285492,
+      "learning_rate": 9.995568825714479e-06,
+      "loss": 0.3623,
+      "step": 130,
+      "student_loss": 0.000613231910392642,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03892733564013841,
+      "grad_norm": 11.875,
+      "kd_loss": 0.4139902889728546,
+      "learning_rate": 9.994360021283235e-06,
+      "loss": 0.3269,
+      "step": 135,
+      "student_loss": 0.059860486537218094,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.040369088811995385,
+      "grad_norm": 17.0,
+      "kd_loss": 0.15953019261360168,
+      "learning_rate": 9.993005693541098e-06,
+      "loss": 0.2964,
+      "step": 140,
+      "student_loss": 0.0005189953953959048,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04181084198385236,
+      "grad_norm": 19.5,
+      "kd_loss": 0.7493298053741455,
+      "learning_rate": 9.991505881949837e-06,
+      "loss": 0.3058,
+      "step": 145,
+      "student_loss": 0.053292322903871536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04325259515570934,
+      "grad_norm": 15.8125,
+      "kd_loss": 0.09782249480485916,
+      "learning_rate": 9.989860630210251e-06,
+      "loss": 0.3046,
+      "step": 150,
+      "student_loss": 0.0007303969468921423,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04469434832756632,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.119688019156456,
+      "learning_rate": 9.98806998626091e-06,
+      "loss": 0.3088,
+      "step": 155,
+      "student_loss": 0.0005232616094872355,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0461361014994233,
+      "grad_norm": 17.625,
+      "kd_loss": 0.2501586973667145,
+      "learning_rate": 9.98613400227676e-06,
+      "loss": 0.3508,
+      "step": 160,
+      "student_loss": 0.0909157246351242,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04757785467128028,
+      "grad_norm": 25.75,
+      "kd_loss": 0.23997722566127777,
+      "learning_rate": 9.984052734667586e-06,
+      "loss": 0.309,
+      "step": 165,
+      "student_loss": 0.0003585258382372558,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.049019607843137254,
+      "grad_norm": 53.75,
+      "kd_loss": 0.08952173590660095,
+      "learning_rate": 9.981826244076387e-06,
+      "loss": 0.2906,
+      "step": 170,
+      "student_loss": 0.0005561846191994846,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05046136101499423,
+      "grad_norm": 40.0,
+      "kd_loss": 0.22894923388957977,
+      "learning_rate": 9.979454595377594e-06,
+      "loss": 0.2992,
+      "step": 175,
+      "student_loss": 0.0037626794073730707,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05190311418685121,
+      "grad_norm": 23.75,
+      "kd_loss": 0.23231007158756256,
+      "learning_rate": 9.97693785767519e-06,
+      "loss": 0.278,
+      "step": 180,
+      "student_loss": 0.0004788874357473105,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05334486735870819,
+      "grad_norm": 32.25,
+      "kd_loss": 0.32399287819862366,
+      "learning_rate": 9.9742761043007e-06,
+      "loss": 0.3037,
+      "step": 185,
+      "student_loss": 0.05988827347755432,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05478662053056517,
+      "grad_norm": 18.75,
+      "kd_loss": 0.46008631587028503,
+      "learning_rate": 9.971469412811032e-06,
+      "loss": 0.2833,
+      "step": 190,
+      "student_loss": 0.0008702761260792613,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.056228373702422146,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.41200897097587585,
+      "learning_rate": 9.968517864986247e-06,
+      "loss": 0.2848,
+      "step": 195,
+      "student_loss": 0.01731002703309059,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05767012687427912,
+      "grad_norm": 10.875,
+      "kd_loss": 0.17150147259235382,
+      "learning_rate": 9.965421546827153e-06,
+      "loss": 0.2805,
+      "step": 200,
+      "student_loss": 0.0014504786813631654,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0591118800461361,
+      "grad_norm": 20.875,
+      "kd_loss": 0.1884479522705078,
+      "learning_rate": 9.962180548552812e-06,
+      "loss": 0.2697,
+      "step": 205,
+      "student_loss": 0.04227042570710182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06055363321799308,
+      "grad_norm": 23.75,
+      "kd_loss": 0.24175123870372772,
+      "learning_rate": 9.958794964597906e-06,
+      "loss": 0.2654,
+      "step": 210,
+      "student_loss": 0.0005445634596981108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.061995386389850055,
+      "grad_norm": 25.875,
+      "kd_loss": 0.2347751259803772,
+      "learning_rate": 9.955264893609986e-06,
+      "loss": 0.294,
+      "step": 215,
+      "student_loss": 0.0005844403640367091,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06343713956170703,
+      "grad_norm": 17.25,
+      "kd_loss": 0.34608691930770874,
+      "learning_rate": 9.951590438446597e-06,
+      "loss": 0.2777,
+      "step": 220,
+      "student_loss": 0.06770455092191696,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06487889273356401,
+      "grad_norm": 37.0,
+      "kd_loss": 0.13642175495624542,
+      "learning_rate": 9.947771706172285e-06,
+      "loss": 0.2711,
+      "step": 225,
+      "student_loss": 0.029741473495960236,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06632064590542099,
+      "grad_norm": 13.5,
+      "kd_loss": 0.24233467876911163,
+      "learning_rate": 9.943808808055472e-06,
+      "loss": 0.2623,
+      "step": 230,
+      "student_loss": 0.0009022229351103306,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06776239907727798,
+      "grad_norm": 11.875,
+      "kd_loss": 0.6708654165267944,
+      "learning_rate": 9.93970185956522e-06,
+      "loss": 0.2862,
+      "step": 235,
+      "student_loss": 0.019873904064297676,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06920415224913495,
+      "grad_norm": 28.0,
+      "kd_loss": 0.18894419074058533,
+      "learning_rate": 9.93545098036786e-06,
+      "loss": 0.2625,
+      "step": 240,
+      "student_loss": 0.0006731516914442182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07064590542099193,
+      "grad_norm": 20.25,
+      "kd_loss": 0.13373719155788422,
+      "learning_rate": 9.931056294323507e-06,
+      "loss": 0.2638,
+      "step": 245,
+      "student_loss": 0.0005933287902735174,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07208765859284891,
+      "grad_norm": 10.0,
+      "kd_loss": 0.12812890112400055,
+      "learning_rate": 9.926517929482454e-06,
+      "loss": 0.2926,
+      "step": 250,
+      "student_loss": 0.00034213275648653507,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07352941176470588,
+      "grad_norm": 16.75,
+      "kd_loss": 0.08018118143081665,
+      "learning_rate": 9.921836018081438e-06,
+      "loss": 0.2428,
+      "step": 255,
+      "student_loss": 0.00025593984173610806,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07497116493656286,
+      "grad_norm": 10.75,
+      "kd_loss": 0.19167569279670715,
+      "learning_rate": 9.917010696539793e-06,
+      "loss": 0.305,
+      "step": 260,
+      "student_loss": 0.04356108978390694,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07641291810841984,
+      "grad_norm": 26.625,
+      "kd_loss": 0.0696549043059349,
+      "learning_rate": 9.912042105455462e-06,
+      "loss": 0.2747,
+      "step": 265,
+      "student_loss": 0.0003771705087274313,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07785467128027682,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.06586078554391861,
+      "learning_rate": 9.906930389600917e-06,
+      "loss": 0.2801,
+      "step": 270,
+      "student_loss": 0.00041506686829961836,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07929642445213379,
+      "grad_norm": 20.75,
+      "kd_loss": 0.14978329837322235,
+      "learning_rate": 9.901675697918925e-06,
+      "loss": 0.2886,
+      "step": 275,
+      "student_loss": 0.04765838012099266,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08073817762399077,
+      "grad_norm": 21.75,
+      "kd_loss": 0.17263588309288025,
+      "learning_rate": 9.896278183518216e-06,
+      "loss": 0.2897,
+      "step": 280,
+      "student_loss": 0.0004967491258867085,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08217993079584775,
+      "grad_norm": 20.0,
+      "kd_loss": 0.10580945014953613,
+      "learning_rate": 9.890738003669029e-06,
+      "loss": 0.293,
+      "step": 285,
+      "student_loss": 0.0005849753506481647,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08362168396770472,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.09176977723836899,
+      "learning_rate": 9.885055319798514e-06,
+      "loss": 0.3088,
+      "step": 290,
+      "student_loss": 0.0006238222122192383,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0850634371395617,
+      "grad_norm": 10.9375,
+      "kd_loss": 0.07891599833965302,
+      "learning_rate": 9.879230297486034e-06,
+      "loss": 0.2957,
+      "step": 295,
+      "student_loss": 0.0005840823869220912,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08650519031141868,
+      "grad_norm": 7.9375,
+      "kd_loss": 0.3616674840450287,
+      "learning_rate": 9.873263106458353e-06,
+      "loss": 0.2483,
+      "step": 300,
+      "student_loss": 0.019933484494686127,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08794694348327567,
+      "grad_norm": 15.9375,
+      "kd_loss": 0.15585829317569733,
+      "learning_rate": 9.86715392058467e-06,
+      "loss": 0.2522,
+      "step": 305,
+      "student_loss": 0.018561875447630882,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08938869665513265,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.06951466202735901,
+      "learning_rate": 9.860902917871566e-06,
+      "loss": 0.2646,
+      "step": 310,
+      "student_loss": 0.00046623480739071965,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09083044982698962,
+      "grad_norm": 11.5,
+      "kd_loss": 0.2836111783981323,
+      "learning_rate": 9.854510280457819e-06,
+      "loss": 0.2393,
+      "step": 315,
+      "student_loss": 0.012322928756475449,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0922722029988466,
+      "grad_norm": 13.6875,
+      "kd_loss": 0.10447105020284653,
+      "learning_rate": 9.847976194609083e-06,
+      "loss": 0.2668,
+      "step": 320,
+      "student_loss": 0.00048153125680983067,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09371395617070358,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.4263528883457184,
+      "learning_rate": 9.841300850712479e-06,
+      "loss": 0.2377,
+      "step": 325,
+      "student_loss": 0.00044430175330489874,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09515570934256055,
+      "grad_norm": 17.5,
+      "kd_loss": 0.6019083261489868,
+      "learning_rate": 9.834484443271035e-06,
+      "loss": 0.2697,
+      "step": 330,
+      "student_loss": 0.05738024041056633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09659746251441753,
+      "grad_norm": 7.125,
+      "kd_loss": 0.0715835765004158,
+      "learning_rate": 9.827527170898019e-06,
+      "loss": 0.2054,
+      "step": 335,
+      "student_loss": 0.0005960066919215024,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09803921568627451,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.10987883061170578,
+      "learning_rate": 9.820429236311158e-06,
+      "loss": 0.2695,
+      "step": 340,
+      "student_loss": 0.009573950432240963,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09948096885813149,
+      "grad_norm": 34.0,
+      "kd_loss": 0.17168864607810974,
+      "learning_rate": 9.81319084632673e-06,
+      "loss": 0.258,
+      "step": 345,
+      "student_loss": 0.016774671152234077,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10092272202998846,
+      "grad_norm": 21.375,
+      "kd_loss": 0.532006025314331,
+      "learning_rate": 9.805812211853534e-06,
+      "loss": 0.2368,
+      "step": 350,
+      "student_loss": 0.0042114779353141785,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10236447520184544,
+      "grad_norm": 16.625,
+      "kd_loss": 0.0480809211730957,
+      "learning_rate": 9.798293547886748e-06,
+      "loss": 0.2384,
+      "step": 355,
+      "student_loss": 0.000468328595161438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10380622837370242,
+      "grad_norm": 19.625,
+      "kd_loss": 0.06214166805148125,
+      "learning_rate": 9.790635073501662e-06,
+      "loss": 0.2473,
+      "step": 360,
+      "student_loss": 0.0004774518311023712,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1052479815455594,
+      "grad_norm": 11.0,
+      "kd_loss": 0.23296360671520233,
+      "learning_rate": 9.782837011847294e-06,
+      "loss": 0.3018,
+      "step": 365,
+      "student_loss": 0.045026302337646484,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10668973471741638,
+      "grad_norm": 14.5625,
+      "kd_loss": 0.39091649651527405,
+      "learning_rate": 9.774899590139897e-06,
+      "loss": 0.2487,
+      "step": 370,
+      "student_loss": 0.019704369828104973,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10813148788927336,
+      "grad_norm": 9.1875,
+      "kd_loss": 0.1680285483598709,
+      "learning_rate": 9.766823039656328e-06,
+      "loss": 0.2398,
+      "step": 375,
+      "student_loss": 0.038442861288785934,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10957324106113034,
+      "grad_norm": 17.875,
+      "kd_loss": 0.06734355539083481,
+      "learning_rate": 9.758607595727309e-06,
+      "loss": 0.2636,
+      "step": 380,
+      "student_loss": 0.0004427676321938634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11101499423298732,
+      "grad_norm": 16.25,
+      "kd_loss": 0.16070111095905304,
+      "learning_rate": 9.75025349773058e-06,
+      "loss": 0.2488,
+      "step": 385,
+      "student_loss": 0.0003361160634085536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11245674740484429,
+      "grad_norm": 10.0,
+      "kd_loss": 0.06088823080062866,
+      "learning_rate": 9.741760989083914e-06,
+      "loss": 0.2351,
+      "step": 390,
+      "student_loss": 0.00025634359917603433,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11389850057670127,
+      "grad_norm": 17.375,
+      "kd_loss": 0.1013672724366188,
+      "learning_rate": 9.73313031723803e-06,
+      "loss": 0.2348,
+      "step": 395,
+      "student_loss": 0.00046920194290578365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11534025374855825,
+      "grad_norm": 12.9375,
+      "kd_loss": 0.12313513457775116,
+      "learning_rate": 9.724361733669383e-06,
+      "loss": 0.1916,
+      "step": 400,
+      "student_loss": 0.0006792352651245892,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11678200692041522,
+      "grad_norm": 12.4375,
+      "kd_loss": 0.11874714493751526,
+      "learning_rate": 9.715455493872827e-06,
+      "loss": 0.2445,
+      "step": 405,
+      "student_loss": 0.0060541993007063866,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1182237600922722,
+      "grad_norm": 15.875,
+      "kd_loss": 0.3652341663837433,
+      "learning_rate": 9.70641185735419e-06,
+      "loss": 0.2546,
+      "step": 410,
+      "student_loss": 0.017243457958102226,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11966551326412918,
+      "grad_norm": 14.0,
+      "kd_loss": 0.1578780561685562,
+      "learning_rate": 9.697231087622691e-06,
+      "loss": 0.2759,
+      "step": 415,
+      "student_loss": 0.0010528146522119641,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12110726643598616,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.3285977840423584,
+      "learning_rate": 9.687913452183277e-06,
+      "loss": 0.2341,
+      "step": 420,
+      "student_loss": 0.06141834706068039,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12254901960784313,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.05784276872873306,
+      "learning_rate": 9.678459222528825e-06,
+      "loss": 0.2234,
+      "step": 425,
+      "student_loss": 0.0011342725483700633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12399077277970011,
+      "grad_norm": 11.8125,
+      "kd_loss": 0.35227036476135254,
+      "learning_rate": 9.668868674132224e-06,
+      "loss": 0.3045,
+      "step": 430,
+      "student_loss": 0.05136450007557869,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1254325259515571,
+      "grad_norm": 13.0625,
+      "kd_loss": 0.09926026314496994,
+      "learning_rate": 9.65914208643836e-06,
+      "loss": 0.2054,
+      "step": 435,
+      "student_loss": 0.0007687933393754065,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12687427912341406,
+      "grad_norm": 10.8125,
+      "kd_loss": 0.3320644795894623,
+      "learning_rate": 9.649279742855964e-06,
+      "loss": 0.2635,
+      "step": 440,
+      "student_loss": 0.02549891546368599,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12831603229527105,
+      "grad_norm": 10.0,
+      "kd_loss": 0.19906648993492126,
+      "learning_rate": 9.639281930749363e-06,
+      "loss": 0.2341,
+      "step": 445,
+      "student_loss": 0.025979263707995415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12975778546712802,
+      "grad_norm": 9.25,
+      "kd_loss": 0.164601668715477,
+      "learning_rate": 9.629148941430096e-06,
+      "loss": 0.2751,
+      "step": 450,
+      "student_loss": 0.0007244550506584346,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.131199538638985,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.04286447539925575,
+      "learning_rate": 9.618881070148436e-06,
+      "loss": 0.2417,
+      "step": 455,
+      "student_loss": 0.000796566077042371,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13264129181084197,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.17657798528671265,
+      "learning_rate": 9.608478616084784e-06,
+      "loss": 0.1985,
+      "step": 460,
+      "student_loss": 0.00047529107541777194,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13408304498269896,
+      "grad_norm": 6.90625,
+      "kd_loss": 0.2981259524822235,
+      "learning_rate": 9.597941882340947e-06,
+      "loss": 0.2786,
+      "step": 465,
+      "student_loss": 0.00040858631837181747,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13552479815455595,
+      "grad_norm": 25.0,
+      "kd_loss": 0.09742365032434464,
+      "learning_rate": 9.587271175931313e-06,
+      "loss": 0.2365,
+      "step": 470,
+      "student_loss": 0.0003401920839678496,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13696655132641292,
+      "grad_norm": 23.625,
+      "kd_loss": 0.24699804186820984,
+      "learning_rate": 9.5764668077739e-06,
+      "loss": 0.1986,
+      "step": 475,
+      "student_loss": 0.0005240993923507631,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1384083044982699,
+      "grad_norm": 21.75,
+      "kd_loss": 0.19472193717956543,
+      "learning_rate": 9.5655290926813e-06,
+      "loss": 0.2274,
+      "step": 480,
+      "student_loss": 0.008671707473695278,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13985005767012687,
+      "grad_norm": 17.125,
+      "kd_loss": 0.055427636951208115,
+      "learning_rate": 9.554458349351507e-06,
+      "loss": 0.2293,
+      "step": 485,
+      "student_loss": 0.0014853382017463446,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14129181084198386,
+      "grad_norm": 13.3125,
+      "kd_loss": 0.2377857267856598,
+      "learning_rate": 9.54325490035863e-06,
+      "loss": 0.2595,
+      "step": 490,
+      "student_loss": 0.000622899504378438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14273356401384082,
+      "grad_norm": 9.3125,
+      "kd_loss": 0.3495025932788849,
+      "learning_rate": 9.531919072143488e-06,
+      "loss": 0.2468,
+      "step": 495,
+      "student_loss": 0.0005211852840147913,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14417531718569782,
+      "grad_norm": 10.3125,
+      "kd_loss": 0.18743589520454407,
+      "learning_rate": 9.520451195004107e-06,
+      "loss": 0.2461,
+      "step": 500,
+      "student_loss": 0.0002995518152602017,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14561707035755478,
+      "grad_norm": 12.5,
+      "kd_loss": 0.23420903086662292,
+      "learning_rate": 9.508851603086094e-06,
+      "loss": 0.2539,
+      "step": 505,
+      "student_loss": 0.0004655766242649406,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14705882352941177,
+      "grad_norm": 20.375,
+      "kd_loss": 0.06832753121852875,
+      "learning_rate": 9.497120634372894e-06,
+      "loss": 0.261,
+      "step": 510,
+      "student_loss": 0.0003136875166092068,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14850057670126873,
+      "grad_norm": 8.9375,
+      "kd_loss": 0.09611348807811737,
+      "learning_rate": 9.48525863067595e-06,
+      "loss": 0.2116,
+      "step": 515,
+      "student_loss": 0.007644087541848421,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14994232987312572,
+      "grad_norm": 7.28125,
+      "kd_loss": 0.2377752661705017,
+      "learning_rate": 9.473265937624748e-06,
+      "loss": 0.217,
+      "step": 520,
+      "student_loss": 0.04185197129845619,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1513840830449827,
+      "grad_norm": 23.375,
+      "kd_loss": 0.20080943405628204,
+      "learning_rate": 9.461142904656722e-06,
+      "loss": 0.249,
+      "step": 525,
+      "student_loss": 0.000431343651143834,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15282583621683968,
+      "grad_norm": 13.5625,
+      "kd_loss": 0.0958016961812973,
+      "learning_rate": 9.448889885007105e-06,
+      "loss": 0.2305,
+      "step": 530,
+      "student_loss": 0.0003980971814598888,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15426758938869667,
+      "grad_norm": 21.625,
+      "kd_loss": 0.37275564670562744,
+      "learning_rate": 9.436507235698613e-06,
+      "loss": 0.2309,
+      "step": 535,
+      "student_loss": 0.0004774174012709409,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15570934256055363,
+      "grad_norm": 31.375,
+      "kd_loss": 0.17777006328105927,
+      "learning_rate": 9.423995317531049e-06,
+      "loss": 0.2529,
+      "step": 540,
+      "student_loss": 0.0005782285006716847,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15715109573241062,
+      "grad_norm": 17.125,
+      "kd_loss": 0.07950557768344879,
+      "learning_rate": 9.4113544950708e-06,
+      "loss": 0.2076,
+      "step": 545,
+      "student_loss": 0.00357666308991611,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.15859284890426759,
+      "grad_norm": 17.0,
+      "kd_loss": 0.15328699350357056,
+      "learning_rate": 9.398585136640195e-06,
+      "loss": 0.2046,
+      "step": 550,
+      "student_loss": 0.0001933475286932662,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16003460207612458,
+      "grad_norm": 20.25,
+      "kd_loss": 0.5593510866165161,
+      "learning_rate": 9.38568761430679e-06,
+      "loss": 0.2422,
+      "step": 555,
+      "student_loss": 0.06740490347146988,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16147635524798154,
+      "grad_norm": 16.875,
+      "kd_loss": 0.4548700153827667,
+      "learning_rate": 9.372662303872514e-06,
+      "loss": 0.2419,
+      "step": 560,
+      "student_loss": 0.0748598724603653,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16291810841983853,
+      "grad_norm": 9.375,
+      "kd_loss": 0.22124871611595154,
+      "learning_rate": 9.359509584862735e-06,
+      "loss": 0.2165,
+      "step": 565,
+      "student_loss": 0.03530263155698776,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1643598615916955,
+      "grad_norm": 17.875,
+      "kd_loss": 0.26488637924194336,
+      "learning_rate": 9.346229840515186e-06,
+      "loss": 0.2562,
+      "step": 570,
+      "student_loss": 0.010391860269010067,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16580161476355249,
+      "grad_norm": 15.3125,
+      "kd_loss": 0.09221018850803375,
+      "learning_rate": 9.332823457768805e-06,
+      "loss": 0.2131,
+      "step": 575,
+      "student_loss": 0.0004648648900911212,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16724336793540945,
+      "grad_norm": 9.625,
+      "kd_loss": 0.153616264462471,
+      "learning_rate": 9.31929082725246e-06,
+      "loss": 0.205,
+      "step": 580,
+      "student_loss": 0.0011936463415622711,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.16868512110726644,
+      "grad_norm": 37.75,
+      "kd_loss": 0.15198436379432678,
+      "learning_rate": 9.305632343273567e-06,
+      "loss": 0.1994,
+      "step": 585,
+      "student_loss": 0.0003124019131064415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1701268742791234,
+      "grad_norm": 9.75,
+      "kd_loss": 0.17398428916931152,
+      "learning_rate": 9.291848403806604e-06,
+      "loss": 0.2032,
+      "step": 590,
+      "student_loss": 0.0004771619278471917,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1715686274509804,
+      "grad_norm": 17.25,
+      "kd_loss": 0.10845454782247543,
+      "learning_rate": 9.277939410481507e-06,
+      "loss": 0.2295,
+      "step": 595,
+      "student_loss": 0.0003602717479225248,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17301038062283736,
+      "grad_norm": 21.75,
+      "kd_loss": 0.2581179738044739,
+      "learning_rate": 9.263905768571977e-06,
+      "loss": 0.2434,
+      "step": 600,
+      "student_loss": 0.04208357632160187,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17445213379469435,
+      "grad_norm": 20.75,
+      "kd_loss": 0.19223031401634216,
+      "learning_rate": 9.249747886983664e-06,
+      "loss": 0.2328,
+      "step": 605,
+      "student_loss": 0.07404639571905136,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.17589388696655134,
+      "grad_norm": 13.5625,
+      "kd_loss": 0.3590433597564697,
+      "learning_rate": 9.235466178242255e-06,
+      "loss": 0.238,
+      "step": 610,
+      "student_loss": 0.0011161614675074816,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1773356401384083,
+      "grad_norm": 26.625,
+      "kd_loss": 0.06018394976854324,
+      "learning_rate": 9.221061058481456e-06,
+      "loss": 0.1984,
+      "step": 615,
+      "student_loss": 0.0002010497555602342,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1787773933102653,
+      "grad_norm": 16.5,
+      "kd_loss": 0.2592211663722992,
+      "learning_rate": 9.206532947430865e-06,
+      "loss": 0.2219,
+      "step": 620,
+      "student_loss": 0.013502068817615509,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18021914648212226,
+      "grad_norm": 16.0,
+      "kd_loss": 0.09846622496843338,
+      "learning_rate": 9.191882268403743e-06,
+      "loss": 0.2017,
+      "step": 625,
+      "student_loss": 0.0058509972877800465,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18166089965397925,
+      "grad_norm": 27.75,
+      "kd_loss": 0.5237650871276855,
+      "learning_rate": 9.177109448284673e-06,
+      "loss": 0.236,
+      "step": 630,
+      "student_loss": 0.06524383276700974,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1831026528258362,
+      "grad_norm": 18.0,
+      "kd_loss": 0.13129356503486633,
+      "learning_rate": 9.162214917517139e-06,
+      "loss": 0.2477,
+      "step": 635,
+      "student_loss": 0.00046817411202937365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1845444059976932,
+      "grad_norm": 12.125,
+      "kd_loss": 0.09964244067668915,
+      "learning_rate": 9.14719911009096e-06,
+      "loss": 0.1964,
+      "step": 640,
+      "student_loss": 0.00038446439430117607,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18598615916955016,
+      "grad_norm": 12.3125,
+      "kd_loss": 0.1809924840927124,
+      "learning_rate": 9.132062463529666e-06,
+      "loss": 0.2034,
+      "step": 645,
+      "student_loss": 0.0003987185191363096,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18742791234140715,
+      "grad_norm": 43.75,
+      "kd_loss": 0.07568997144699097,
+      "learning_rate": 9.11680541887774e-06,
+      "loss": 0.2124,
+      "step": 650,
+      "student_loss": 0.0005847765132784843,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.18886966551326412,
+      "grad_norm": 32.0,
+      "kd_loss": 0.18582190573215485,
+      "learning_rate": 9.101428420687759e-06,
+      "loss": 0.2401,
+      "step": 655,
+      "student_loss": 0.00038169475737959146,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1903114186851211,
+      "grad_norm": 28.75,
+      "kd_loss": 0.1885007619857788,
+      "learning_rate": 9.085931917007464e-06,
+      "loss": 0.2095,
+      "step": 660,
+      "student_loss": 0.0004360108287073672,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19175317185697807,
+      "grad_norm": 16.625,
+      "kd_loss": 0.4099441170692444,
+      "learning_rate": 9.070316359366681e-06,
+      "loss": 0.2286,
+      "step": 665,
+      "student_loss": 0.0005011683679185808,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19319492502883506,
+      "grad_norm": 13.75,
+      "kd_loss": 0.2379698008298874,
+      "learning_rate": 9.054582202764175e-06,
+      "loss": 0.202,
+      "step": 670,
+      "student_loss": 0.0059415721334517,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19463667820069205,
+      "grad_norm": 7.5,
+      "kd_loss": 0.05046012997627258,
+      "learning_rate": 9.038729905654393e-06,
+      "loss": 0.2227,
+      "step": 675,
+      "student_loss": 0.00044423757935874164,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19607843137254902,
+      "grad_norm": 14.125,
+      "kd_loss": 0.2490309476852417,
+      "learning_rate": 9.022759929934109e-06,
+      "loss": 0.1973,
+      "step": 680,
+      "student_loss": 0.07809463143348694,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.197520184544406,
+      "grad_norm": 19.25,
+      "kd_loss": 0.24093234539031982,
+      "learning_rate": 9.006672740928952e-06,
+      "loss": 0.2393,
+      "step": 685,
+      "student_loss": 0.044076789170503616,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.19896193771626297,
+      "grad_norm": 19.25,
+      "kd_loss": 0.7235217094421387,
+      "learning_rate": 8.990468807379861e-06,
+      "loss": 0.2368,
+      "step": 690,
+      "student_loss": 0.054187677800655365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20040369088811996,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.11449181288480759,
+      "learning_rate": 8.974148601429426e-06,
+      "loss": 0.2614,
+      "step": 695,
+      "student_loss": 0.0049613662995398045,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20184544405997693,
+      "grad_norm": 10.625,
+      "kd_loss": 0.06947001069784164,
+      "learning_rate": 8.957712598608123e-06,
+      "loss": 0.2167,
+      "step": 700,
+      "student_loss": 0.0008642565226182342,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20328719723183392,
+      "grad_norm": 12.5625,
+      "kd_loss": 0.2155514508485794,
+      "learning_rate": 8.941161277820462e-06,
+      "loss": 0.2222,
+      "step": 705,
+      "student_loss": 0.0004866976523771882,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20472895040369088,
+      "grad_norm": 11.25,
+      "kd_loss": 0.45056015253067017,
+      "learning_rate": 8.924495121331038e-06,
+      "loss": 0.2276,
+      "step": 710,
+      "student_loss": 0.008678597398102283,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20617070357554787,
+      "grad_norm": 23.875,
+      "kd_loss": 0.08582590520381927,
+      "learning_rate": 8.907714614750473e-06,
+      "loss": 0.2311,
+      "step": 715,
+      "student_loss": 0.01213450264185667,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20761245674740483,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.16561608016490936,
+      "learning_rate": 8.890820247021263e-06,
+      "loss": 0.23,
+      "step": 720,
+      "student_loss": 0.002681850688531995,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.20905420991926182,
+      "grad_norm": 20.875,
+      "kd_loss": 0.07642826437950134,
+      "learning_rate": 8.873812510403545e-06,
+      "loss": 0.1851,
+      "step": 725,
+      "student_loss": 0.00026688360958360136,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2104959630911188,
+      "grad_norm": 17.25,
+      "kd_loss": 0.17072460055351257,
+      "learning_rate": 8.85669190046074e-06,
+      "loss": 0.2456,
+      "step": 730,
+      "student_loss": 0.017637869343161583,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21193771626297578,
+      "grad_norm": 13.25,
+      "kd_loss": 0.1058090403676033,
+      "learning_rate": 8.839458916045117e-06,
+      "loss": 0.2502,
+      "step": 735,
+      "student_loss": 0.0003498170990496874,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21337946943483277,
+      "grad_norm": 6.5,
+      "kd_loss": 0.2888850271701813,
+      "learning_rate": 8.822114059283267e-06,
+      "loss": 0.2033,
+      "step": 740,
+      "student_loss": 0.002009955933317542,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21482122260668973,
+      "grad_norm": 7.625,
+      "kd_loss": 0.11317810416221619,
+      "learning_rate": 8.804657835561456e-06,
+      "loss": 0.2178,
+      "step": 745,
+      "student_loss": 0.04010392725467682,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21626297577854672,
+      "grad_norm": 17.5,
+      "kd_loss": 0.7453857064247131,
+      "learning_rate": 8.787090753510917e-06,
+      "loss": 0.1926,
+      "step": 750,
+      "student_loss": 0.024328170344233513,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2177047289504037,
+      "grad_norm": 10.6875,
+      "kd_loss": 0.06280636787414551,
+      "learning_rate": 8.76941332499301e-06,
+      "loss": 0.2399,
+      "step": 755,
+      "student_loss": 0.032401587814092636,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.21914648212226068,
+      "grad_norm": 11.6875,
+      "kd_loss": 0.17805851995944977,
+      "learning_rate": 8.751626065084328e-06,
+      "loss": 0.2135,
+      "step": 760,
+      "student_loss": 0.022408630698919296,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22058823529411764,
+      "grad_norm": 10.0625,
+      "kd_loss": 0.3160823881626129,
+      "learning_rate": 8.733729492061675e-06,
+      "loss": 0.206,
+      "step": 765,
+      "student_loss": 0.005015719681978226,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22202998846597463,
+      "grad_norm": 9.9375,
+      "kd_loss": 0.31819650530815125,
+      "learning_rate": 8.715724127386971e-06,
+      "loss": 0.2282,
+      "step": 770,
+      "student_loss": 0.0012502025347203016,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2234717416378316,
+      "grad_norm": 20.25,
+      "kd_loss": 0.31151849031448364,
+      "learning_rate": 8.697610495692055e-06,
+      "loss": 0.1895,
+      "step": 775,
+      "student_loss": 0.0014239229494705796,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22491349480968859,
+      "grad_norm": 13.0625,
+      "kd_loss": 0.2508423626422882,
+      "learning_rate": 8.679389124763396e-06,
+      "loss": 0.2453,
+      "step": 780,
+      "student_loss": 0.0013981886440888047,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22635524798154555,
+      "grad_norm": 20.875,
+      "kd_loss": 0.04912475869059563,
+      "learning_rate": 8.661060545526724e-06,
+      "loss": 0.2078,
+      "step": 785,
+      "student_loss": 0.00030144155607558787,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.22779700115340254,
+      "grad_norm": 18.25,
+      "kd_loss": 0.05954095348715782,
+      "learning_rate": 8.64262529203155e-06,
+      "loss": 0.1789,
+      "step": 790,
+      "student_loss": 0.001504220417700708,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2292387543252595,
+      "grad_norm": 8.1875,
+      "kd_loss": 0.1065668910741806,
+      "learning_rate": 8.624083901435613e-06,
+      "loss": 0.2176,
+      "step": 795,
+      "student_loss": 0.00033886852907016873,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2306805074971165,
+      "grad_norm": 16.125,
+      "kd_loss": 0.5510667562484741,
+      "learning_rate": 8.60543691398922e-06,
+      "loss": 0.2246,
+      "step": 800,
+      "student_loss": 0.06318172067403793,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23212226066897348,
+      "grad_norm": 12.5,
+      "kd_loss": 0.0667741522192955,
+      "learning_rate": 8.586684873019513e-06,
+      "loss": 0.2272,
+      "step": 805,
+      "student_loss": 0.0003553811402525753,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23356401384083045,
+      "grad_norm": 8.875,
+      "kd_loss": 0.12136233597993851,
+      "learning_rate": 8.567828324914633e-06,
+      "loss": 0.1892,
+      "step": 810,
+      "student_loss": 0.0004278223786968738,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23500576701268744,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.11104825884103775,
+      "learning_rate": 8.548867819107803e-06,
+      "loss": 0.2218,
+      "step": 815,
+      "student_loss": 0.0006955342250876129,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2364475201845444,
+      "grad_norm": 8.625,
+      "kd_loss": 0.21142087876796722,
+      "learning_rate": 8.52980390806131e-06,
+      "loss": 0.1921,
+      "step": 820,
+      "student_loss": 0.007228876929730177,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2378892733564014,
+      "grad_norm": 10.875,
+      "kd_loss": 0.17360462248325348,
+      "learning_rate": 8.51063714725042e-06,
+      "loss": 0.2136,
+      "step": 825,
+      "student_loss": 0.0012437300756573677,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.23933102652825836,
+      "grad_norm": 9.625,
+      "kd_loss": 0.3183634877204895,
+      "learning_rate": 8.49136809514719e-06,
+      "loss": 0.2384,
+      "step": 830,
+      "student_loss": 0.004646586254239082,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24077277970011535,
+      "grad_norm": 17.25,
+      "kd_loss": 0.17037151753902435,
+      "learning_rate": 8.471997313204183e-06,
+      "loss": 0.1873,
+      "step": 835,
+      "student_loss": 0.015499490313231945,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2422145328719723,
+      "grad_norm": 18.0,
+      "kd_loss": 0.14555610716342926,
+      "learning_rate": 8.452525365838123e-06,
+      "loss": 0.2059,
+      "step": 840,
+      "student_loss": 0.0005742039065808058,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2436562860438293,
+      "grad_norm": 15.8125,
+      "kd_loss": 0.06973056495189667,
+      "learning_rate": 8.432952820413447e-06,
+      "loss": 0.2197,
+      "step": 845,
+      "student_loss": 0.0005098649999126792,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24509803921568626,
+      "grad_norm": 10.625,
+      "kd_loss": 0.2858434021472931,
+      "learning_rate": 8.41328024722577e-06,
+      "loss": 0.2115,
+      "step": 850,
+      "student_loss": 0.012342527508735657,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24653979238754326,
+      "grad_norm": 15.0625,
+      "kd_loss": 0.09209071844816208,
+      "learning_rate": 8.393508219485263e-06,
+      "loss": 0.2095,
+      "step": 855,
+      "student_loss": 0.00024691459839232266,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.24798154555940022,
+      "grad_norm": 8.625,
+      "kd_loss": 0.08900707215070724,
+      "learning_rate": 8.373637313299966e-06,
+      "loss": 0.2341,
+      "step": 860,
+      "student_loss": 0.0005091276834718883,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2494232987312572,
+      "grad_norm": 18.125,
+      "kd_loss": 0.11508733034133911,
+      "learning_rate": 8.353668107658984e-06,
+      "loss": 0.2099,
+      "step": 865,
+      "student_loss": 0.0002668049419298768,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2508650519031142,
+      "grad_norm": 15.4375,
+      "kd_loss": 0.29352444410324097,
+      "learning_rate": 8.333601184415632e-06,
+      "loss": 0.2173,
+      "step": 870,
+      "student_loss": 0.0016936544561758637,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25230680507497116,
+      "grad_norm": 9.25,
+      "kd_loss": 0.05814331769943237,
+      "learning_rate": 8.31343712827047e-06,
+      "loss": 0.2007,
+      "step": 875,
+      "student_loss": 0.00016406318172812462,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2537485582468281,
+      "grad_norm": 8.0625,
+      "kd_loss": 0.12616939842700958,
+      "learning_rate": 8.293176526754274e-06,
+      "loss": 0.2079,
+      "step": 880,
+      "student_loss": 0.00044341504690237343,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25519031141868515,
+      "grad_norm": 9.875,
+      "kd_loss": 0.6110478639602661,
+      "learning_rate": 8.27281997021091e-06,
+      "loss": 0.1749,
+      "step": 885,
+      "student_loss": 0.03696923702955246,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2566320645905421,
+      "grad_norm": 12.875,
+      "kd_loss": 0.049601469188928604,
+      "learning_rate": 8.25236805178014e-06,
+      "loss": 0.2115,
+      "step": 890,
+      "student_loss": 0.00036779773654416203,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25807381776239907,
+      "grad_norm": 7.46875,
+      "kd_loss": 0.3071838617324829,
+      "learning_rate": 8.231821367380335e-06,
+      "loss": 0.2182,
+      "step": 895,
+      "student_loss": 0.0032658062409609556,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.25951557093425603,
+      "grad_norm": 19.25,
+      "kd_loss": 0.1950993835926056,
+      "learning_rate": 8.211180515691108e-06,
+      "loss": 0.2058,
+      "step": 900,
+      "student_loss": 0.03542155399918556,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26095732410611305,
+      "grad_norm": 12.125,
+      "kd_loss": 0.4699344038963318,
+      "learning_rate": 8.190446098135883e-06,
+      "loss": 0.1775,
+      "step": 905,
+      "student_loss": 0.04188324883580208,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26239907727797,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.04896146059036255,
+      "learning_rate": 8.16961871886435e-06,
+      "loss": 0.1931,
+      "step": 910,
+      "student_loss": 0.00029122710111550987,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.263840830449827,
+      "grad_norm": 23.5,
+      "kd_loss": 0.514575183391571,
+      "learning_rate": 8.148698984734887e-06,
+      "loss": 0.1831,
+      "step": 915,
+      "student_loss": 0.09658902883529663,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26528258362168394,
+      "grad_norm": 20.125,
+      "kd_loss": 0.16474592685699463,
+      "learning_rate": 8.127687505296852e-06,
+      "loss": 0.2031,
+      "step": 920,
+      "student_loss": 0.020180951803922653,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.26672433679354096,
+      "grad_norm": 13.6875,
+      "kd_loss": 0.03160429745912552,
+      "learning_rate": 8.106584892772844e-06,
+      "loss": 0.1953,
+      "step": 925,
+      "student_loss": 0.0002911738120019436,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2681660899653979,
+      "grad_norm": 9.875,
+      "kd_loss": 0.09231647849082947,
+      "learning_rate": 8.08539176204085e-06,
+      "loss": 0.1953,
+      "step": 930,
+      "student_loss": 0.02371821738779545,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2696078431372549,
+      "grad_norm": 23.875,
+      "kd_loss": 0.11771639436483383,
+      "learning_rate": 8.064108730616336e-06,
+      "loss": 0.1856,
+      "step": 935,
+      "student_loss": 0.0008747431566007435,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2710495963091119,
+      "grad_norm": 14.9375,
+      "kd_loss": 0.24315571784973145,
+      "learning_rate": 8.042736418634252e-06,
+      "loss": 0.2151,
+      "step": 940,
+      "student_loss": 0.011635007336735725,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27249134948096887,
+      "grad_norm": 14.1875,
+      "kd_loss": 0.08452239632606506,
+      "learning_rate": 8.02127544883096e-06,
+      "loss": 0.2101,
+      "step": 945,
+      "student_loss": 0.0033563401084393263,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27393310265282583,
+      "grad_norm": 19.5,
+      "kd_loss": 0.12136533856391907,
+      "learning_rate": 7.999726446526095e-06,
+      "loss": 0.1898,
+      "step": 950,
+      "student_loss": 0.0002544254530221224,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2753748558246828,
+      "grad_norm": 18.25,
+      "kd_loss": 0.12915752828121185,
+      "learning_rate": 7.978090039604342e-06,
+      "loss": 0.1979,
+      "step": 955,
+      "student_loss": 0.0007313934620469809,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2768166089965398,
+      "grad_norm": 10.375,
+      "kd_loss": 0.16751284897327423,
+      "learning_rate": 7.956366858497138e-06,
+      "loss": 0.1886,
+      "step": 960,
+      "student_loss": 0.0003871220105793327,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2782583621683968,
+      "grad_norm": 9.6875,
+      "kd_loss": 0.25768253207206726,
+      "learning_rate": 7.934557536164304e-06,
+      "loss": 0.1998,
+      "step": 965,
+      "student_loss": 0.013239127583801746,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.27970011534025374,
+      "grad_norm": 25.5,
+      "kd_loss": 0.11466177552938461,
+      "learning_rate": 7.91266270807561e-06,
+      "loss": 0.202,
+      "step": 970,
+      "student_loss": 0.0003071602841373533,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2811418685121107,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.08647243678569794,
+      "learning_rate": 7.890683012192249e-06,
+      "loss": 0.2151,
+      "step": 975,
+      "student_loss": 0.017678463831543922,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2825836216839677,
+      "grad_norm": 11.0,
+      "kd_loss": 0.09852449595928192,
+      "learning_rate": 7.868619088948246e-06,
+      "loss": 0.191,
+      "step": 980,
+      "student_loss": 0.00038518354995176196,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2840253748558247,
+      "grad_norm": 11.25,
+      "kd_loss": 0.09779661893844604,
+      "learning_rate": 7.846471581231814e-06,
+      "loss": 0.2316,
+      "step": 985,
+      "student_loss": 0.00031637900974601507,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.28546712802768165,
+      "grad_norm": 14.0,
+      "kd_loss": 0.08365006744861603,
+      "learning_rate": 7.824241134366604e-06,
+      "loss": 0.1693,
+      "step": 990,
+      "student_loss": 0.00032978563103824854,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2869088811995386,
+      "grad_norm": 20.0,
+      "kd_loss": 0.29525840282440186,
+      "learning_rate": 7.801928396092908e-06,
+      "loss": 0.2244,
+      "step": 995,
+      "student_loss": 0.00505430344492197,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.28835063437139563,
+      "grad_norm": 7.53125,
+      "kd_loss": 0.1885785460472107,
+      "learning_rate": 7.779534016548791e-06,
+      "loss": 0.2048,
+      "step": 1000,
+      "student_loss": 0.0076581151224672794,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2897923875432526,
+      "grad_norm": 26.625,
+      "kd_loss": 0.6930287480354309,
+      "learning_rate": 7.75705864825114e-06,
+      "loss": 0.2184,
+      "step": 1005,
+      "student_loss": 0.026617757976055145,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.29123414071510956,
+      "grad_norm": 9.5,
+      "kd_loss": 0.03928455710411072,
+      "learning_rate": 7.734502946076656e-06,
+      "loss": 0.2054,
+      "step": 1010,
+      "student_loss": 0.0003597455797716975,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2926758938869666,
+      "grad_norm": 9.625,
+      "kd_loss": 0.49502038955688477,
+      "learning_rate": 7.711867567242769e-06,
+      "loss": 0.2299,
+      "step": 1015,
+      "student_loss": 0.017246609553694725,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.29411764705882354,
+      "grad_norm": 19.375,
+      "kd_loss": 0.38125079870224,
+      "learning_rate": 7.689153171288487e-06,
+      "loss": 0.2001,
+      "step": 1020,
+      "student_loss": 0.04544902592897415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2955594002306805,
+      "grad_norm": 14.9375,
+      "kd_loss": 0.08253186196088791,
+      "learning_rate": 7.666360420055188e-06,
+      "loss": 0.1866,
+      "step": 1025,
+      "student_loss": 0.0003647907287813723,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.29700115340253747,
+      "grad_norm": 13.625,
+      "kd_loss": 0.2902968227863312,
+      "learning_rate": 7.643489977667327e-06,
+      "loss": 0.2117,
+      "step": 1030,
+      "student_loss": 0.004529732279479504,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.2984429065743945,
+      "grad_norm": 10.375,
+      "kd_loss": 0.15247558057308197,
+      "learning_rate": 7.6205425105130855e-06,
+      "loss": 0.1843,
+      "step": 1035,
+      "student_loss": 0.0018669666023924947,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.29988465974625145,
+      "grad_norm": 16.0,
+      "kd_loss": 0.045563094317913055,
+      "learning_rate": 7.597518687224959e-06,
+      "loss": 0.2043,
+      "step": 1040,
+      "student_loss": 0.0007197121740318835,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3013264129181084,
+      "grad_norm": 15.9375,
+      "kd_loss": 0.03541402891278267,
+      "learning_rate": 7.574419178660269e-06,
+      "loss": 0.1969,
+      "step": 1045,
+      "student_loss": 0.00030927587067708373,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3027681660899654,
+      "grad_norm": 9.0625,
+      "kd_loss": 0.09471673518419266,
+      "learning_rate": 7.551244657881618e-06,
+      "loss": 0.2233,
+      "step": 1050,
+      "student_loss": 0.04248344525694847,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3042099192618224,
+      "grad_norm": 14.0,
+      "kd_loss": 0.07426086813211441,
+      "learning_rate": 7.527995800137287e-06,
+      "loss": 0.1966,
+      "step": 1055,
+      "student_loss": 0.0003190846764482558,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.30565167243367936,
+      "grad_norm": 24.125,
+      "kd_loss": 0.17531871795654297,
+      "learning_rate": 7.504673282841544e-06,
+      "loss": 0.2017,
+      "step": 1060,
+      "student_loss": 0.01057682279497385,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3070934256055363,
+      "grad_norm": 9.3125,
+      "kd_loss": 0.7109049558639526,
+      "learning_rate": 7.481277785554918e-06,
+      "loss": 0.1868,
+      "step": 1065,
+      "student_loss": 0.04248557239770889,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.30853517877739334,
+      "grad_norm": 20.875,
+      "kd_loss": 0.22240523993968964,
+      "learning_rate": 7.457809989964393e-06,
+      "loss": 0.2062,
+      "step": 1070,
+      "student_loss": 0.00877230241894722,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3099769319492503,
+      "grad_norm": 26.0,
+      "kd_loss": 0.04868102818727493,
+      "learning_rate": 7.434270579863549e-06,
+      "loss": 0.2274,
+      "step": 1075,
+      "student_loss": 0.004425506107509136,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.31141868512110726,
+      "grad_norm": 5.8125,
+      "kd_loss": 0.06708519905805588,
+      "learning_rate": 7.4106602411326345e-06,
+      "loss": 0.1883,
+      "step": 1080,
+      "student_loss": 0.00029492450994439423,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3128604382929642,
+      "grad_norm": 17.375,
+      "kd_loss": 0.04687422141432762,
+      "learning_rate": 7.386979661718585e-06,
+      "loss": 0.2363,
+      "step": 1085,
+      "student_loss": 0.0002493698848411441,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.31430219146482125,
+      "grad_norm": 17.5,
+      "kd_loss": 0.4828321635723114,
+      "learning_rate": 7.363229531614973e-06,
+      "loss": 0.201,
+      "step": 1090,
+      "student_loss": 0.0029170920606702566,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3157439446366782,
+      "grad_norm": 8.9375,
+      "kd_loss": 0.10699938982725143,
+      "learning_rate": 7.339410542841906e-06,
+      "loss": 0.22,
+      "step": 1095,
+      "student_loss": 0.0637359470129013,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.31718569780853517,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.1359812468290329,
+      "learning_rate": 7.315523389425867e-06,
+      "loss": 0.2056,
+      "step": 1100,
+      "student_loss": 0.00031933377613313496,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.31862745098039214,
+      "grad_norm": 6.9375,
+      "kd_loss": 0.22178106009960175,
+      "learning_rate": 7.291568767379484e-06,
+      "loss": 0.1688,
+      "step": 1105,
+      "student_loss": 0.0017254141857847571,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.32006920415224915,
+      "grad_norm": 39.25,
+      "kd_loss": 0.1460317075252533,
+      "learning_rate": 7.267547374681259e-06,
+      "loss": 0.2073,
+      "step": 1110,
+      "student_loss": 0.0006291258032433689,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3215109573241061,
+      "grad_norm": 17.125,
+      "kd_loss": 0.12435565888881683,
+      "learning_rate": 7.24345991125522e-06,
+      "loss": 0.2099,
+      "step": 1115,
+      "student_loss": 0.0006149607943370938,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3229527104959631,
+      "grad_norm": 12.9375,
+      "kd_loss": 0.23137590289115906,
+      "learning_rate": 7.219307078950536e-06,
+      "loss": 0.1876,
+      "step": 1120,
+      "student_loss": 0.000807489559520036,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.32439446366782004,
+      "grad_norm": 13.3125,
+      "kd_loss": 0.24867931008338928,
+      "learning_rate": 7.195089581521064e-06,
+      "loss": 0.1902,
+      "step": 1125,
+      "student_loss": 0.03634409233927727,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.32583621683967706,
+      "grad_norm": 14.25,
+      "kd_loss": 0.30060166120529175,
+      "learning_rate": 7.170808124604842e-06,
+      "loss": 0.1663,
+      "step": 1130,
+      "student_loss": 0.01190225314348936,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.327277970011534,
+      "grad_norm": 16.875,
+      "kd_loss": 0.3193962275981903,
+      "learning_rate": 7.14646341570353e-06,
+      "loss": 0.2296,
+      "step": 1135,
+      "student_loss": 0.0074716294184327126,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.328719723183391,
+      "grad_norm": 14.75,
+      "kd_loss": 0.0424165204167366,
+      "learning_rate": 7.122056164161795e-06,
+      "loss": 0.2025,
+      "step": 1140,
+      "student_loss": 0.0025572802405804396,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.330161476355248,
+      "grad_norm": 23.875,
+      "kd_loss": 0.05634481459856033,
+      "learning_rate": 7.097587081146636e-06,
+      "loss": 0.178,
+      "step": 1145,
+      "student_loss": 0.0013154952321201563,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.33160322952710497,
+      "grad_norm": 7.84375,
+      "kd_loss": 0.06748944520950317,
+      "learning_rate": 7.073056879626681e-06,
+      "loss": 0.2012,
+      "step": 1150,
+      "student_loss": 0.000519739231094718,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.33304498269896193,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.09680163860321045,
+      "learning_rate": 7.048466274351389e-06,
+      "loss": 0.1812,
+      "step": 1155,
+      "student_loss": 0.02215045690536499,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3344867358708189,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.1690923124551773,
+      "learning_rate": 7.023815981830236e-06,
+      "loss": 0.1998,
+      "step": 1160,
+      "student_loss": 0.03653755784034729,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3359284890426759,
+      "grad_norm": 19.25,
+      "kd_loss": 0.054503124207258224,
+      "learning_rate": 6.999106720311846e-06,
+      "loss": 0.1847,
+      "step": 1165,
+      "student_loss": 0.016527529805898666,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3373702422145329,
+      "grad_norm": 25.25,
+      "kd_loss": 0.11392180621623993,
+      "learning_rate": 6.974339209763043e-06,
+      "loss": 0.1652,
+      "step": 1170,
+      "student_loss": 0.021370824426412582,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.33881199538638984,
+      "grad_norm": 24.0,
+      "kd_loss": 0.3750007748603821,
+      "learning_rate": 6.949514171847891e-06,
+      "loss": 0.1876,
+      "step": 1175,
+      "student_loss": 0.00037981336936354637,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3402537485582468,
+      "grad_norm": 20.875,
+      "kd_loss": 0.2402525544166565,
+      "learning_rate": 6.924632329906657e-06,
+      "loss": 0.2015,
+      "step": 1180,
+      "student_loss": 0.0009415193926542997,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3416955017301038,
+      "grad_norm": 13.75,
+      "kd_loss": 0.09193167835474014,
+      "learning_rate": 6.899694408934734e-06,
+      "loss": 0.2006,
+      "step": 1185,
+      "student_loss": 0.002855707658454776,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3431372549019608,
+      "grad_norm": 7.9375,
+      "kd_loss": 0.11176048219203949,
+      "learning_rate": 6.874701135561524e-06,
+      "loss": 0.2076,
+      "step": 1190,
+      "student_loss": 0.000517252366989851,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.34457900807381775,
+      "grad_norm": 12.3125,
+      "kd_loss": 0.10253634303808212,
+      "learning_rate": 6.849653238029261e-06,
+      "loss": 0.186,
+      "step": 1195,
+      "student_loss": 0.0019411896355450153,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3460207612456747,
+      "grad_norm": 11.5,
+      "kd_loss": 0.038868214935064316,
+      "learning_rate": 6.824551446171788e-06,
+      "loss": 0.1987,
+      "step": 1200,
+      "student_loss": 0.00023505830904468894,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.34746251441753173,
+      "grad_norm": 8.0625,
+      "kd_loss": 0.1182335838675499,
+      "learning_rate": 6.7993964913932975e-06,
+      "loss": 0.1752,
+      "step": 1205,
+      "student_loss": 0.013255883008241653,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3489042675893887,
+      "grad_norm": 9.5,
+      "kd_loss": 0.15362422168254852,
+      "learning_rate": 6.774189106647021e-06,
+      "loss": 0.1923,
+      "step": 1210,
+      "student_loss": 0.001112818717956543,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.35034602076124566,
+      "grad_norm": 7.53125,
+      "kd_loss": 0.24585981667041779,
+      "learning_rate": 6.748930026413865e-06,
+      "loss": 0.21,
+      "step": 1215,
+      "student_loss": 0.009471859782934189,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3517877739331027,
+      "grad_norm": 16.0,
+      "kd_loss": 0.26234984397888184,
+      "learning_rate": 6.7236199866810185e-06,
+      "loss": 0.2124,
+      "step": 1220,
+      "student_loss": 0.030534446239471436,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.35322952710495964,
+      "grad_norm": 7.6875,
+      "kd_loss": 0.0947137251496315,
+      "learning_rate": 6.698259724920503e-06,
+      "loss": 0.2052,
+      "step": 1225,
+      "student_loss": 0.0012586602242663503,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3546712802768166,
+      "grad_norm": 19.25,
+      "kd_loss": 0.04720739647746086,
+      "learning_rate": 6.672849980067685e-06,
+      "loss": 0.1804,
+      "step": 1230,
+      "student_loss": 0.0003573665744625032,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.35611303344867357,
+      "grad_norm": 25.625,
+      "kd_loss": 0.09224828332662582,
+      "learning_rate": 6.647391492499746e-06,
+      "loss": 0.1781,
+      "step": 1235,
+      "student_loss": 0.000400040065869689,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3575547866205306,
+      "grad_norm": 9.75,
+      "kd_loss": 0.056848179548978806,
+      "learning_rate": 6.621885004014113e-06,
+      "loss": 0.2324,
+      "step": 1240,
+      "student_loss": 0.0004060330393258482,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.35899653979238755,
+      "grad_norm": 13.375,
+      "kd_loss": 0.07238893955945969,
+      "learning_rate": 6.596331257806837e-06,
+      "loss": 0.1958,
+      "step": 1245,
+      "student_loss": 0.0004042874788865447,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3604382929642445,
+      "grad_norm": 25.25,
+      "kd_loss": 0.18240241706371307,
+      "learning_rate": 6.570730998450945e-06,
+      "loss": 0.2142,
+      "step": 1250,
+      "student_loss": 0.026042336598038673,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3618800461361015,
+      "grad_norm": 19.375,
+      "kd_loss": 0.05622217431664467,
+      "learning_rate": 6.545084971874738e-06,
+      "loss": 0.1991,
+      "step": 1255,
+      "student_loss": 0.00731692323461175,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3633217993079585,
+      "grad_norm": 7.3125,
+      "kd_loss": 0.046645376831293106,
+      "learning_rate": 6.519393925340067e-06,
+      "loss": 0.2078,
+      "step": 1260,
+      "student_loss": 0.0001817115262383595,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.36476355247981546,
+      "grad_norm": 15.75,
+      "kd_loss": 0.15666288137435913,
+      "learning_rate": 6.49365860742055e-06,
+      "loss": 0.1982,
+      "step": 1265,
+      "student_loss": 0.018377086147665977,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3662053056516724,
+      "grad_norm": 23.75,
+      "kd_loss": 0.06302918493747711,
+      "learning_rate": 6.467879767979764e-06,
+      "loss": 0.1924,
+      "step": 1270,
+      "student_loss": 0.00037668898585252464,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.36764705882352944,
+      "grad_norm": 16.875,
+      "kd_loss": 0.2142418473958969,
+      "learning_rate": 6.442058158149396e-06,
+      "loss": 0.1684,
+      "step": 1275,
+      "student_loss": 0.006484415847808123,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3690888119953864,
+      "grad_norm": 9.1875,
+      "kd_loss": 0.18019038438796997,
+      "learning_rate": 6.4161945303073535e-06,
+      "loss": 0.1779,
+      "step": 1280,
+      "student_loss": 0.0010251402854919434,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.37053056516724336,
+      "grad_norm": 9.4375,
+      "kd_loss": 0.08422896265983582,
+      "learning_rate": 6.390289638055851e-06,
+      "loss": 0.1814,
+      "step": 1285,
+      "student_loss": 0.00045269622933119535,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3719723183391003,
+      "grad_norm": 12.4375,
+      "kd_loss": 0.12106428295373917,
+      "learning_rate": 6.364344236199441e-06,
+      "loss": 0.1848,
+      "step": 1290,
+      "student_loss": 0.044445283710956573,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.37341407151095735,
+      "grad_norm": 14.4375,
+      "kd_loss": 0.06391695141792297,
+      "learning_rate": 6.3383590807230264e-06,
+      "loss": 0.2084,
+      "step": 1295,
+      "student_loss": 0.0003671850427053869,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3748558246828143,
+      "grad_norm": 11.8125,
+      "kd_loss": 0.15638460218906403,
+      "learning_rate": 6.3123349287698345e-06,
+      "loss": 0.1934,
+      "step": 1300,
+      "student_loss": 0.00037028471706435084,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3762975778546713,
+      "grad_norm": 29.0,
+      "kd_loss": 0.27376165986061096,
+      "learning_rate": 6.286272538619351e-06,
+      "loss": 0.1877,
+      "step": 1305,
+      "student_loss": 0.000540899345651269,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.37773933102652824,
+      "grad_norm": 7.34375,
+      "kd_loss": 0.06797322630882263,
+      "learning_rate": 6.260172669665233e-06,
+      "loss": 0.1694,
+      "step": 1310,
+      "student_loss": 0.0005134029779583216,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.37918108419838525,
+      "grad_norm": 17.375,
+      "kd_loss": 0.12302712351083755,
+      "learning_rate": 6.234036082393171e-06,
+      "loss": 0.1863,
+      "step": 1315,
+      "student_loss": 0.001553699024952948,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3806228373702422,
+      "grad_norm": 10.0625,
+      "kd_loss": 0.10804786533117294,
+      "learning_rate": 6.207863538358741e-06,
+      "loss": 0.1749,
+      "step": 1320,
+      "student_loss": 0.0014063200214877725,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3820645905420992,
+      "grad_norm": 7.40625,
+      "kd_loss": 0.23416215181350708,
+      "learning_rate": 6.181655800165207e-06,
+      "loss": 0.1978,
+      "step": 1325,
+      "student_loss": 0.0003445672045927495,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.38350634371395614,
+      "grad_norm": 10.4375,
+      "kd_loss": 0.23408842086791992,
+      "learning_rate": 6.155413631441307e-06,
+      "loss": 0.2035,
+      "step": 1330,
+      "student_loss": 0.009326261468231678,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.38494809688581316,
+      "grad_norm": 10.9375,
+      "kd_loss": 0.07795803993940353,
+      "learning_rate": 6.129137796818997e-06,
+      "loss": 0.175,
+      "step": 1335,
+      "student_loss": 0.0004052516014780849,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3863898500576701,
+      "grad_norm": 5.6875,
+      "kd_loss": 0.3068411648273468,
+      "learning_rate": 6.102829061911176e-06,
+      "loss": 0.1948,
+      "step": 1340,
+      "student_loss": 0.020378023386001587,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3878316032295271,
+      "grad_norm": 7.34375,
+      "kd_loss": 0.05054645612835884,
+      "learning_rate": 6.076488193289375e-06,
+      "loss": 0.1878,
+      "step": 1345,
+      "student_loss": 0.0004019946791231632,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3892733564013841,
+      "grad_norm": 12.75,
+      "kd_loss": 0.21713578701019287,
+      "learning_rate": 6.050115958461423e-06,
+      "loss": 0.1808,
+      "step": 1350,
+      "student_loss": 0.015930594876408577,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.39071510957324107,
+      "grad_norm": 15.3125,
+      "kd_loss": 0.45843416452407837,
+      "learning_rate": 6.02371312584908e-06,
+      "loss": 0.1704,
+      "step": 1355,
+      "student_loss": 0.02067803405225277,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.39215686274509803,
+      "grad_norm": 8.5625,
+      "kd_loss": 0.25011980533599854,
+      "learning_rate": 5.997280464765655e-06,
+      "loss": 0.1897,
+      "step": 1360,
+      "student_loss": 0.0008441711543127894,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.393598615916955,
+      "grad_norm": 9.8125,
+      "kd_loss": 0.38673579692840576,
+      "learning_rate": 5.970818745393579e-06,
+      "loss": 0.2267,
+      "step": 1365,
+      "student_loss": 0.08822725713253021,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.395040369088812,
+      "grad_norm": 19.625,
+      "kd_loss": 0.0780089944601059,
+      "learning_rate": 5.9443287387619754e-06,
+      "loss": 0.1805,
+      "step": 1370,
+      "student_loss": 0.03733369708061218,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.396482122260669,
+      "grad_norm": 10.9375,
+      "kd_loss": 0.45517951250076294,
+      "learning_rate": 5.9178112167241805e-06,
+      "loss": 0.1868,
+      "step": 1375,
+      "student_loss": 0.017787568271160126,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.39792387543252594,
+      "grad_norm": 10.375,
+      "kd_loss": 0.05289127677679062,
+      "learning_rate": 5.8912669519352725e-06,
+      "loss": 0.1822,
+      "step": 1380,
+      "student_loss": 0.0045715938322246075,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.3993656286043829,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.07237643748521805,
+      "learning_rate": 5.864696717829539e-06,
+      "loss": 0.1986,
+      "step": 1385,
+      "student_loss": 0.02439916506409645,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4008073817762399,
+      "grad_norm": 9.1875,
+      "kd_loss": 0.11604559421539307,
+      "learning_rate": 5.838101288597951e-06,
+      "loss": 0.189,
+      "step": 1390,
+      "student_loss": 0.0003213919699192047,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4022491349480969,
+      "grad_norm": 16.875,
+      "kd_loss": 0.4560457170009613,
+      "learning_rate": 5.8114814391656046e-06,
+      "loss": 0.2012,
+      "step": 1395,
+      "student_loss": 0.008913847617805004,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.40369088811995385,
+      "grad_norm": 18.25,
+      "kd_loss": 0.19607780873775482,
+      "learning_rate": 5.78483794516914e-06,
+      "loss": 0.1885,
+      "step": 1400,
+      "student_loss": 0.0004231977218296379,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.40513264129181087,
+      "grad_norm": 26.125,
+      "kd_loss": 0.05656301975250244,
+      "learning_rate": 5.75817158293414e-06,
+      "loss": 0.2286,
+      "step": 1405,
+      "student_loss": 0.0007274498348124325,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.40657439446366783,
+      "grad_norm": 11.9375,
+      "kd_loss": 0.1830921322107315,
+      "learning_rate": 5.731483129452514e-06,
+      "loss": 0.1792,
+      "step": 1410,
+      "student_loss": 0.0005606960039585829,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4080161476355248,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.05641094222664833,
+      "learning_rate": 5.704773362359854e-06,
+      "loss": 0.1803,
+      "step": 1415,
+      "student_loss": 0.004419934470206499,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.40945790080738176,
+      "grad_norm": 9.5625,
+      "kd_loss": 0.1356193721294403,
+      "learning_rate": 5.678043059912776e-06,
+      "loss": 0.1571,
+      "step": 1420,
+      "student_loss": 0.00037621092633344233,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4108996539792388,
+      "grad_norm": 10.5625,
+      "kd_loss": 0.22703546285629272,
+      "learning_rate": 5.6512930009662524e-06,
+      "loss": 0.1959,
+      "step": 1425,
+      "student_loss": 0.025084203109145164,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.41234140715109574,
+      "grad_norm": 7.625,
+      "kd_loss": 0.054302290081977844,
+      "learning_rate": 5.624523964950903e-06,
+      "loss": 0.1822,
+      "step": 1430,
+      "student_loss": 0.00032259186264127493,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4137831603229527,
+      "grad_norm": 14.625,
+      "kd_loss": 0.0907973200082779,
+      "learning_rate": 5.597736731850295e-06,
+      "loss": 0.1848,
+      "step": 1435,
+      "student_loss": 0.00121297559235245,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.41522491349480967,
+      "grad_norm": 17.625,
+      "kd_loss": 0.07823797315359116,
+      "learning_rate": 5.570932082178219e-06,
+      "loss": 0.2208,
+      "step": 1440,
+      "student_loss": 0.021054241806268692,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4166666666666667,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.4457983374595642,
+      "learning_rate": 5.5441107969559315e-06,
+      "loss": 0.1838,
+      "step": 1445,
+      "student_loss": 0.019648732617497444,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.41810841983852365,
+      "grad_norm": 12.8125,
+      "kd_loss": 0.2628614604473114,
+      "learning_rate": 5.517273657689419e-06,
+      "loss": 0.2101,
+      "step": 1450,
+      "student_loss": 0.0026446706615388393,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4195501730103806,
+      "grad_norm": 19.0,
+      "kd_loss": 0.05944440886378288,
+      "learning_rate": 5.490421446346608e-06,
+      "loss": 0.19,
+      "step": 1455,
+      "student_loss": 0.0010834980057552457,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4209919261822376,
+      "grad_norm": 6.84375,
+      "kd_loss": 0.1711495965719223,
+      "learning_rate": 5.463554945334589e-06,
+      "loss": 0.1958,
+      "step": 1460,
+      "student_loss": 0.0003729817399289459,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4224336793540946,
+      "grad_norm": 9.6875,
+      "kd_loss": 0.10369019955396652,
+      "learning_rate": 5.43667493747682e-06,
+      "loss": 0.1707,
+      "step": 1465,
+      "student_loss": 0.014215357601642609,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.42387543252595156,
+      "grad_norm": 12.4375,
+      "kd_loss": 0.1022537350654602,
+      "learning_rate": 5.409782205990317e-06,
+      "loss": 0.1845,
+      "step": 1470,
+      "student_loss": 0.0002443333505652845,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4253171856978085,
+      "grad_norm": 9.9375,
+      "kd_loss": 0.03169335797429085,
+      "learning_rate": 5.3828775344628245e-06,
+      "loss": 0.141,
+      "step": 1475,
+      "student_loss": 0.0003328607708681375,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.42675893886966554,
+      "grad_norm": 15.625,
+      "kd_loss": 0.04210688918828964,
+      "learning_rate": 5.355961706829997e-06,
+      "loss": 0.1892,
+      "step": 1480,
+      "student_loss": 0.0003736468206625432,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.4282006920415225,
+      "grad_norm": 11.625,
+      "kd_loss": 0.028124002739787102,
+      "learning_rate": 5.329035507352548e-06,
+      "loss": 0.1905,
+      "step": 1485,
+      "student_loss": 0.00025713234208524227,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.42964244521337946,
+      "grad_norm": 8.875,
+      "kd_loss": 0.03635227680206299,
+      "learning_rate": 5.3020997205933985e-06,
+      "loss": 0.1642,
+      "step": 1490,
+      "student_loss": 0.0016982207307592034,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.43108419838523643,
+      "grad_norm": 17.25,
+      "kd_loss": 0.07685169577598572,
+      "learning_rate": 5.275155131394825e-06,
+      "loss": 0.1832,
+      "step": 1495,
+      "student_loss": 0.0003758777165785432,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.43252595155709345,
+      "grad_norm": 10.5625,
+      "kd_loss": 0.4748556613922119,
+      "learning_rate": 5.248202524855578e-06,
+      "loss": 0.1758,
+      "step": 1500,
+      "student_loss": 0.013860415667295456,
+      "teacher_loss": 0.0
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|action_sep|>": 151670,
+  "<|arg_sep|>": 151671,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|call_sep|>": 151666,
+  "<|end_of_text|>": 151673,
+  "<|endoftext|>": 151643,
+  "<|exception_sep|>": 151669,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|frame_sep|>": 151672,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|latent_end|>": 151675,
+  "<|latent_start|>": 151674,
+  "<|line_sep|>": 151667,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|return_sep|>": 151668,
+  "<|trace_context_start|>": 151665,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "dtype": "bfloat16",
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151676
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6e112c4cfd365214a2d150c793beecd4941cf90706bb1129412639ccfd95bd
+size 3096275515

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/thought_projector.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42b4c4ec484d4efd9bca68c973c5cf2016cf8ca99b927894e03ce461c1571510
+size 9445953

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83a790d654474f5dfe225f889afd0210313eb1083f942671f2c4b8e95a1c922b
+size 11424004

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/tokenizer_config.json

@@ -0,0 +1,295 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<|trace_context_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|call_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|line_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|return_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|exception_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|action_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|arg_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|frame_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|latent_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|latent_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoints/codi_frozen_1.5b_hidden/checkpoint-500/trainer_state.json

@@ -0,0 +1,1034 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.14417531718569782,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014417531718569781,
+      "grad_norm": 322.0,
+      "kd_loss": 1.997420072555542,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 2.1422,
+      "step": 5,
+      "student_loss": 0.0605761781334877,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0028835063437139563,
+      "grad_norm": 186.0,
+      "kd_loss": 1.8595829010009766,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 2.0829,
+      "step": 10,
+      "student_loss": 0.09249057620763779,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.004325259515570935,
+      "grad_norm": 117.5,
+      "kd_loss": 1.5637085437774658,
+      "learning_rate": 1.5555555555555558e-06,
+      "loss": 1.972,
+      "step": 15,
+      "student_loss": 0.026328546926379204,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0057670126874279125,
+      "grad_norm": 106.5,
+      "kd_loss": 1.4912058115005493,
+      "learning_rate": 2.1111111111111114e-06,
+      "loss": 1.8011,
+      "step": 20,
+      "student_loss": 0.13309946656227112,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00720876585928489,
+      "grad_norm": 86.5,
+      "kd_loss": 1.1542646884918213,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 1.5427,
+      "step": 25,
+      "student_loss": 0.036511313170194626,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.00865051903114187,
+      "grad_norm": 58.5,
+      "kd_loss": 0.9403514266014099,
+      "learning_rate": 3.2222222222222227e-06,
+      "loss": 1.2794,
+      "step": 30,
+      "student_loss": 0.01801167242228985,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.010092272202998846,
+      "grad_norm": 31.5,
+      "kd_loss": 0.6764306426048279,
+      "learning_rate": 3.777777777777778e-06,
+      "loss": 1.0897,
+      "step": 35,
+      "student_loss": 0.0011638500727713108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.011534025374855825,
+      "grad_norm": 27.625,
+      "kd_loss": 0.7702282667160034,
+      "learning_rate": 4.333333333333334e-06,
+      "loss": 0.9024,
+      "step": 40,
+      "student_loss": 0.08332191407680511,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.012975778546712802,
+      "grad_norm": 20.875,
+      "kd_loss": 1.050789475440979,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.7206,
+      "step": 45,
+      "student_loss": 0.07726961374282837,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01441753171856978,
+      "grad_norm": 44.25,
+      "kd_loss": 0.4379478096961975,
+      "learning_rate": 5.444444444444445e-06,
+      "loss": 0.6754,
+      "step": 50,
+      "student_loss": 0.007436605170369148,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.015859284890426758,
+      "grad_norm": 17.5,
+      "kd_loss": 0.3016382157802582,
+      "learning_rate": 6e-06,
+      "loss": 0.5857,
+      "step": 55,
+      "student_loss": 0.002162981079891324,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.01730103806228374,
+      "grad_norm": 24.625,
+      "kd_loss": 0.2297545075416565,
+      "learning_rate": 6.555555555555556e-06,
+      "loss": 0.5265,
+      "step": 60,
+      "student_loss": 0.012505847960710526,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.018742791234140715,
+      "grad_norm": 14.75,
+      "kd_loss": 0.2053602635860443,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 0.5101,
+      "step": 65,
+      "student_loss": 0.04744679108262062,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.020184544405997693,
+      "grad_norm": 34.0,
+      "kd_loss": 0.7437167763710022,
+      "learning_rate": 7.666666666666667e-06,
+      "loss": 0.4523,
+      "step": 70,
+      "student_loss": 0.029973715543746948,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02162629757785467,
+      "grad_norm": 14.0625,
+      "kd_loss": 0.18727970123291016,
+      "learning_rate": 8.222222222222222e-06,
+      "loss": 0.4534,
+      "step": 75,
+      "student_loss": 0.001686512609012425,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02306805074971165,
+      "grad_norm": 31.0,
+      "kd_loss": 0.18602637946605682,
+      "learning_rate": 8.777777777777778e-06,
+      "loss": 0.4338,
+      "step": 80,
+      "student_loss": 0.0014192159287631512,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.024509803921568627,
+      "grad_norm": 14.75,
+      "kd_loss": 0.11737873405218124,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.4378,
+      "step": 85,
+      "student_loss": 0.000477277091704309,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.025951557093425604,
+      "grad_norm": 17.125,
+      "kd_loss": 0.7024360299110413,
+      "learning_rate": 9.88888888888889e-06,
+      "loss": 0.3816,
+      "step": 90,
+      "student_loss": 0.027783483266830444,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.027393310265282585,
+      "grad_norm": 19.125,
+      "kd_loss": 0.3916384279727936,
+      "learning_rate": 9.99995337991255e-06,
+      "loss": 0.3905,
+      "step": 95,
+      "student_loss": 0.013136160559952259,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.02883506343713956,
+      "grad_norm": 13.9375,
+      "kd_loss": 0.09685378521680832,
+      "learning_rate": 9.999763987297266e-06,
+      "loss": 0.3485,
+      "step": 100,
+      "student_loss": 0.0006464557372964919,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03027681660899654,
+      "grad_norm": 12.5,
+      "kd_loss": 0.48202934861183167,
+      "learning_rate": 9.999428913912876e-06,
+      "loss": 0.3555,
+      "step": 105,
+      "student_loss": 0.005315156653523445,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.031718569780853516,
+      "grad_norm": 9.8125,
+      "kd_loss": 0.42474475502967834,
+      "learning_rate": 9.998948169522587e-06,
+      "loss": 0.3616,
+      "step": 110,
+      "student_loss": 0.0304458886384964,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03316032295271049,
+      "grad_norm": 11.9375,
+      "kd_loss": 0.30219101905822754,
+      "learning_rate": 9.998321768134101e-06,
+      "loss": 0.3669,
+      "step": 115,
+      "student_loss": 0.0007903591031208634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03460207612456748,
+      "grad_norm": 47.0,
+      "kd_loss": 0.14825306832790375,
+      "learning_rate": 9.997549727999212e-06,
+      "loss": 0.3014,
+      "step": 120,
+      "student_loss": 0.0007933085435070097,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.036043829296424454,
+      "grad_norm": 10.6875,
+      "kd_loss": 0.6828317046165466,
+      "learning_rate": 9.996632071613257e-06,
+      "loss": 0.3416,
+      "step": 125,
+      "student_loss": 0.056072596460580826,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03748558246828143,
+      "grad_norm": 21.5,
+      "kd_loss": 0.16518603265285492,
+      "learning_rate": 9.995568825714479e-06,
+      "loss": 0.3623,
+      "step": 130,
+      "student_loss": 0.000613231910392642,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.03892733564013841,
+      "grad_norm": 11.875,
+      "kd_loss": 0.4139902889728546,
+      "learning_rate": 9.994360021283235e-06,
+      "loss": 0.3269,
+      "step": 135,
+      "student_loss": 0.059860486537218094,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.040369088811995385,
+      "grad_norm": 17.0,
+      "kd_loss": 0.15953019261360168,
+      "learning_rate": 9.993005693541098e-06,
+      "loss": 0.2964,
+      "step": 140,
+      "student_loss": 0.0005189953953959048,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04181084198385236,
+      "grad_norm": 19.5,
+      "kd_loss": 0.7493298053741455,
+      "learning_rate": 9.991505881949837e-06,
+      "loss": 0.3058,
+      "step": 145,
+      "student_loss": 0.053292322903871536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04325259515570934,
+      "grad_norm": 15.8125,
+      "kd_loss": 0.09782249480485916,
+      "learning_rate": 9.989860630210251e-06,
+      "loss": 0.3046,
+      "step": 150,
+      "student_loss": 0.0007303969468921423,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04469434832756632,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.119688019156456,
+      "learning_rate": 9.98806998626091e-06,
+      "loss": 0.3088,
+      "step": 155,
+      "student_loss": 0.0005232616094872355,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0461361014994233,
+      "grad_norm": 17.625,
+      "kd_loss": 0.2501586973667145,
+      "learning_rate": 9.98613400227676e-06,
+      "loss": 0.3508,
+      "step": 160,
+      "student_loss": 0.0909157246351242,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.04757785467128028,
+      "grad_norm": 25.75,
+      "kd_loss": 0.23997722566127777,
+      "learning_rate": 9.984052734667586e-06,
+      "loss": 0.309,
+      "step": 165,
+      "student_loss": 0.0003585258382372558,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.049019607843137254,
+      "grad_norm": 53.75,
+      "kd_loss": 0.08952173590660095,
+      "learning_rate": 9.981826244076387e-06,
+      "loss": 0.2906,
+      "step": 170,
+      "student_loss": 0.0005561846191994846,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05046136101499423,
+      "grad_norm": 40.0,
+      "kd_loss": 0.22894923388957977,
+      "learning_rate": 9.979454595377594e-06,
+      "loss": 0.2992,
+      "step": 175,
+      "student_loss": 0.0037626794073730707,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05190311418685121,
+      "grad_norm": 23.75,
+      "kd_loss": 0.23231007158756256,
+      "learning_rate": 9.97693785767519e-06,
+      "loss": 0.278,
+      "step": 180,
+      "student_loss": 0.0004788874357473105,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05334486735870819,
+      "grad_norm": 32.25,
+      "kd_loss": 0.32399287819862366,
+      "learning_rate": 9.9742761043007e-06,
+      "loss": 0.3037,
+      "step": 185,
+      "student_loss": 0.05988827347755432,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05478662053056517,
+      "grad_norm": 18.75,
+      "kd_loss": 0.46008631587028503,
+      "learning_rate": 9.971469412811032e-06,
+      "loss": 0.2833,
+      "step": 190,
+      "student_loss": 0.0008702761260792613,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.056228373702422146,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.41200897097587585,
+      "learning_rate": 9.968517864986247e-06,
+      "loss": 0.2848,
+      "step": 195,
+      "student_loss": 0.01731002703309059,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.05767012687427912,
+      "grad_norm": 10.875,
+      "kd_loss": 0.17150147259235382,
+      "learning_rate": 9.965421546827153e-06,
+      "loss": 0.2805,
+      "step": 200,
+      "student_loss": 0.0014504786813631654,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0591118800461361,
+      "grad_norm": 20.875,
+      "kd_loss": 0.1884479522705078,
+      "learning_rate": 9.962180548552812e-06,
+      "loss": 0.2697,
+      "step": 205,
+      "student_loss": 0.04227042570710182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06055363321799308,
+      "grad_norm": 23.75,
+      "kd_loss": 0.24175123870372772,
+      "learning_rate": 9.958794964597906e-06,
+      "loss": 0.2654,
+      "step": 210,
+      "student_loss": 0.0005445634596981108,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.061995386389850055,
+      "grad_norm": 25.875,
+      "kd_loss": 0.2347751259803772,
+      "learning_rate": 9.955264893609986e-06,
+      "loss": 0.294,
+      "step": 215,
+      "student_loss": 0.0005844403640367091,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06343713956170703,
+      "grad_norm": 17.25,
+      "kd_loss": 0.34608691930770874,
+      "learning_rate": 9.951590438446597e-06,
+      "loss": 0.2777,
+      "step": 220,
+      "student_loss": 0.06770455092191696,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06487889273356401,
+      "grad_norm": 37.0,
+      "kd_loss": 0.13642175495624542,
+      "learning_rate": 9.947771706172285e-06,
+      "loss": 0.2711,
+      "step": 225,
+      "student_loss": 0.029741473495960236,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06632064590542099,
+      "grad_norm": 13.5,
+      "kd_loss": 0.24233467876911163,
+      "learning_rate": 9.943808808055472e-06,
+      "loss": 0.2623,
+      "step": 230,
+      "student_loss": 0.0009022229351103306,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06776239907727798,
+      "grad_norm": 11.875,
+      "kd_loss": 0.6708654165267944,
+      "learning_rate": 9.93970185956522e-06,
+      "loss": 0.2862,
+      "step": 235,
+      "student_loss": 0.019873904064297676,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.06920415224913495,
+      "grad_norm": 28.0,
+      "kd_loss": 0.18894419074058533,
+      "learning_rate": 9.93545098036786e-06,
+      "loss": 0.2625,
+      "step": 240,
+      "student_loss": 0.0006731516914442182,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07064590542099193,
+      "grad_norm": 20.25,
+      "kd_loss": 0.13373719155788422,
+      "learning_rate": 9.931056294323507e-06,
+      "loss": 0.2638,
+      "step": 245,
+      "student_loss": 0.0005933287902735174,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07208765859284891,
+      "grad_norm": 10.0,
+      "kd_loss": 0.12812890112400055,
+      "learning_rate": 9.926517929482454e-06,
+      "loss": 0.2926,
+      "step": 250,
+      "student_loss": 0.00034213275648653507,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07352941176470588,
+      "grad_norm": 16.75,
+      "kd_loss": 0.08018118143081665,
+      "learning_rate": 9.921836018081438e-06,
+      "loss": 0.2428,
+      "step": 255,
+      "student_loss": 0.00025593984173610806,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07497116493656286,
+      "grad_norm": 10.75,
+      "kd_loss": 0.19167569279670715,
+      "learning_rate": 9.917010696539793e-06,
+      "loss": 0.305,
+      "step": 260,
+      "student_loss": 0.04356108978390694,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07641291810841984,
+      "grad_norm": 26.625,
+      "kd_loss": 0.0696549043059349,
+      "learning_rate": 9.912042105455462e-06,
+      "loss": 0.2747,
+      "step": 265,
+      "student_loss": 0.0003771705087274313,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07785467128027682,
+      "grad_norm": 12.0625,
+      "kd_loss": 0.06586078554391861,
+      "learning_rate": 9.906930389600917e-06,
+      "loss": 0.2801,
+      "step": 270,
+      "student_loss": 0.00041506686829961836,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.07929642445213379,
+      "grad_norm": 20.75,
+      "kd_loss": 0.14978329837322235,
+      "learning_rate": 9.901675697918925e-06,
+      "loss": 0.2886,
+      "step": 275,
+      "student_loss": 0.04765838012099266,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08073817762399077,
+      "grad_norm": 21.75,
+      "kd_loss": 0.17263588309288025,
+      "learning_rate": 9.896278183518216e-06,
+      "loss": 0.2897,
+      "step": 280,
+      "student_loss": 0.0004967491258867085,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08217993079584775,
+      "grad_norm": 20.0,
+      "kd_loss": 0.10580945014953613,
+      "learning_rate": 9.890738003669029e-06,
+      "loss": 0.293,
+      "step": 285,
+      "student_loss": 0.0005849753506481647,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08362168396770472,
+      "grad_norm": 12.1875,
+      "kd_loss": 0.09176977723836899,
+      "learning_rate": 9.885055319798514e-06,
+      "loss": 0.3088,
+      "step": 290,
+      "student_loss": 0.0006238222122192383,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0850634371395617,
+      "grad_norm": 10.9375,
+      "kd_loss": 0.07891599833965302,
+      "learning_rate": 9.879230297486034e-06,
+      "loss": 0.2957,
+      "step": 295,
+      "student_loss": 0.0005840823869220912,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08650519031141868,
+      "grad_norm": 7.9375,
+      "kd_loss": 0.3616674840450287,
+      "learning_rate": 9.873263106458353e-06,
+      "loss": 0.2483,
+      "step": 300,
+      "student_loss": 0.019933484494686127,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08794694348327567,
+      "grad_norm": 15.9375,
+      "kd_loss": 0.15585829317569733,
+      "learning_rate": 9.86715392058467e-06,
+      "loss": 0.2522,
+      "step": 305,
+      "student_loss": 0.018561875447630882,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.08938869665513265,
+      "grad_norm": 10.1875,
+      "kd_loss": 0.06951466202735901,
+      "learning_rate": 9.860902917871566e-06,
+      "loss": 0.2646,
+      "step": 310,
+      "student_loss": 0.00046623480739071965,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09083044982698962,
+      "grad_norm": 11.5,
+      "kd_loss": 0.2836111783981323,
+      "learning_rate": 9.854510280457819e-06,
+      "loss": 0.2393,
+      "step": 315,
+      "student_loss": 0.012322928756475449,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.0922722029988466,
+      "grad_norm": 13.6875,
+      "kd_loss": 0.10447105020284653,
+      "learning_rate": 9.847976194609083e-06,
+      "loss": 0.2668,
+      "step": 320,
+      "student_loss": 0.00048153125680983067,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09371395617070358,
+      "grad_norm": 11.0625,
+      "kd_loss": 0.4263528883457184,
+      "learning_rate": 9.841300850712479e-06,
+      "loss": 0.2377,
+      "step": 325,
+      "student_loss": 0.00044430175330489874,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09515570934256055,
+      "grad_norm": 17.5,
+      "kd_loss": 0.6019083261489868,
+      "learning_rate": 9.834484443271035e-06,
+      "loss": 0.2697,
+      "step": 330,
+      "student_loss": 0.05738024041056633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09659746251441753,
+      "grad_norm": 7.125,
+      "kd_loss": 0.0715835765004158,
+      "learning_rate": 9.827527170898019e-06,
+      "loss": 0.2054,
+      "step": 335,
+      "student_loss": 0.0005960066919215024,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09803921568627451,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.10987883061170578,
+      "learning_rate": 9.820429236311158e-06,
+      "loss": 0.2695,
+      "step": 340,
+      "student_loss": 0.009573950432240963,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.09948096885813149,
+      "grad_norm": 34.0,
+      "kd_loss": 0.17168864607810974,
+      "learning_rate": 9.81319084632673e-06,
+      "loss": 0.258,
+      "step": 345,
+      "student_loss": 0.016774671152234077,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10092272202998846,
+      "grad_norm": 21.375,
+      "kd_loss": 0.532006025314331,
+      "learning_rate": 9.805812211853534e-06,
+      "loss": 0.2368,
+      "step": 350,
+      "student_loss": 0.0042114779353141785,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10236447520184544,
+      "grad_norm": 16.625,
+      "kd_loss": 0.0480809211730957,
+      "learning_rate": 9.798293547886748e-06,
+      "loss": 0.2384,
+      "step": 355,
+      "student_loss": 0.000468328595161438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10380622837370242,
+      "grad_norm": 19.625,
+      "kd_loss": 0.06214166805148125,
+      "learning_rate": 9.790635073501662e-06,
+      "loss": 0.2473,
+      "step": 360,
+      "student_loss": 0.0004774518311023712,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1052479815455594,
+      "grad_norm": 11.0,
+      "kd_loss": 0.23296360671520233,
+      "learning_rate": 9.782837011847294e-06,
+      "loss": 0.3018,
+      "step": 365,
+      "student_loss": 0.045026302337646484,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10668973471741638,
+      "grad_norm": 14.5625,
+      "kd_loss": 0.39091649651527405,
+      "learning_rate": 9.774899590139897e-06,
+      "loss": 0.2487,
+      "step": 370,
+      "student_loss": 0.019704369828104973,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10813148788927336,
+      "grad_norm": 9.1875,
+      "kd_loss": 0.1680285483598709,
+      "learning_rate": 9.766823039656328e-06,
+      "loss": 0.2398,
+      "step": 375,
+      "student_loss": 0.038442861288785934,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.10957324106113034,
+      "grad_norm": 17.875,
+      "kd_loss": 0.06734355539083481,
+      "learning_rate": 9.758607595727309e-06,
+      "loss": 0.2636,
+      "step": 380,
+      "student_loss": 0.0004427676321938634,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11101499423298732,
+      "grad_norm": 16.25,
+      "kd_loss": 0.16070111095905304,
+      "learning_rate": 9.75025349773058e-06,
+      "loss": 0.2488,
+      "step": 385,
+      "student_loss": 0.0003361160634085536,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11245674740484429,
+      "grad_norm": 10.0,
+      "kd_loss": 0.06088823080062866,
+      "learning_rate": 9.741760989083914e-06,
+      "loss": 0.2351,
+      "step": 390,
+      "student_loss": 0.00025634359917603433,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11389850057670127,
+      "grad_norm": 17.375,
+      "kd_loss": 0.1013672724366188,
+      "learning_rate": 9.73313031723803e-06,
+      "loss": 0.2348,
+      "step": 395,
+      "student_loss": 0.00046920194290578365,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11534025374855825,
+      "grad_norm": 12.9375,
+      "kd_loss": 0.12313513457775116,
+      "learning_rate": 9.724361733669383e-06,
+      "loss": 0.1916,
+      "step": 400,
+      "student_loss": 0.0006792352651245892,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11678200692041522,
+      "grad_norm": 12.4375,
+      "kd_loss": 0.11874714493751526,
+      "learning_rate": 9.715455493872827e-06,
+      "loss": 0.2445,
+      "step": 405,
+      "student_loss": 0.0060541993007063866,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1182237600922722,
+      "grad_norm": 15.875,
+      "kd_loss": 0.3652341663837433,
+      "learning_rate": 9.70641185735419e-06,
+      "loss": 0.2546,
+      "step": 410,
+      "student_loss": 0.017243457958102226,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.11966551326412918,
+      "grad_norm": 14.0,
+      "kd_loss": 0.1578780561685562,
+      "learning_rate": 9.697231087622691e-06,
+      "loss": 0.2759,
+      "step": 415,
+      "student_loss": 0.0010528146522119641,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12110726643598616,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.3285977840423584,
+      "learning_rate": 9.687913452183277e-06,
+      "loss": 0.2341,
+      "step": 420,
+      "student_loss": 0.06141834706068039,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12254901960784313,
+      "grad_norm": 15.1875,
+      "kd_loss": 0.05784276872873306,
+      "learning_rate": 9.678459222528825e-06,
+      "loss": 0.2234,
+      "step": 425,
+      "student_loss": 0.0011342725483700633,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12399077277970011,
+      "grad_norm": 11.8125,
+      "kd_loss": 0.35227036476135254,
+      "learning_rate": 9.668868674132224e-06,
+      "loss": 0.3045,
+      "step": 430,
+      "student_loss": 0.05136450007557869,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1254325259515571,
+      "grad_norm": 13.0625,
+      "kd_loss": 0.09926026314496994,
+      "learning_rate": 9.65914208643836e-06,
+      "loss": 0.2054,
+      "step": 435,
+      "student_loss": 0.0007687933393754065,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12687427912341406,
+      "grad_norm": 10.8125,
+      "kd_loss": 0.3320644795894623,
+      "learning_rate": 9.649279742855964e-06,
+      "loss": 0.2635,
+      "step": 440,
+      "student_loss": 0.02549891546368599,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12831603229527105,
+      "grad_norm": 10.0,
+      "kd_loss": 0.19906648993492126,
+      "learning_rate": 9.639281930749363e-06,
+      "loss": 0.2341,
+      "step": 445,
+      "student_loss": 0.025979263707995415,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.12975778546712802,
+      "grad_norm": 9.25,
+      "kd_loss": 0.164601668715477,
+      "learning_rate": 9.629148941430096e-06,
+      "loss": 0.2751,
+      "step": 450,
+      "student_loss": 0.0007244550506584346,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.131199538638985,
+      "grad_norm": 7.78125,
+      "kd_loss": 0.04286447539925575,
+      "learning_rate": 9.618881070148436e-06,
+      "loss": 0.2417,
+      "step": 455,
+      "student_loss": 0.000796566077042371,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13264129181084197,
+      "grad_norm": 13.1875,
+      "kd_loss": 0.17657798528671265,
+      "learning_rate": 9.608478616084784e-06,
+      "loss": 0.1985,
+      "step": 460,
+      "student_loss": 0.00047529107541777194,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13408304498269896,
+      "grad_norm": 6.90625,
+      "kd_loss": 0.2981259524822235,
+      "learning_rate": 9.597941882340947e-06,
+      "loss": 0.2786,
+      "step": 465,
+      "student_loss": 0.00040858631837181747,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13552479815455595,
+      "grad_norm": 25.0,
+      "kd_loss": 0.09742365032434464,
+      "learning_rate": 9.587271175931313e-06,
+      "loss": 0.2365,
+      "step": 470,
+      "student_loss": 0.0003401920839678496,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13696655132641292,
+      "grad_norm": 23.625,
+      "kd_loss": 0.24699804186820984,
+      "learning_rate": 9.5764668077739e-06,
+      "loss": 0.1986,
+      "step": 475,
+      "student_loss": 0.0005240993923507631,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.1384083044982699,
+      "grad_norm": 21.75,
+      "kd_loss": 0.19472193717956543,
+      "learning_rate": 9.5655290926813e-06,
+      "loss": 0.2274,
+      "step": 480,
+      "student_loss": 0.008671707473695278,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.13985005767012687,
+      "grad_norm": 17.125,
+      "kd_loss": 0.055427636951208115,
+      "learning_rate": 9.554458349351507e-06,
+      "loss": 0.2293,
+      "step": 485,
+      "student_loss": 0.0014853382017463446,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14129181084198386,
+      "grad_norm": 13.3125,
+      "kd_loss": 0.2377857267856598,
+      "learning_rate": 9.54325490035863e-06,
+      "loss": 0.2595,
+      "step": 490,
+      "student_loss": 0.000622899504378438,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14273356401384082,
+      "grad_norm": 9.3125,
+      "kd_loss": 0.3495025932788849,
+      "learning_rate": 9.531919072143488e-06,
+      "loss": 0.2468,
+      "step": 495,
+      "student_loss": 0.0005211852840147913,
+      "teacher_loss": 0.0
+    },
+    {
+      "epoch": 0.14417531718569782,
+      "grad_norm": 10.3125,
+      "kd_loss": 0.18743589520454407,
+      "learning_rate": 9.520451195004107e-06,
+      "loss": 0.2461,
+      "step": 500,
+      "student_loss": 0.0002995518152602017,
+      "teacher_loss": 0.0
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoints/codi_frozen_1.5b_hidden/wandb/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-06-25T13:55:13.216983388+02:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-25T13:55:13.216997191+02:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-06-25T13:55:13.339058881+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.339080395+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.339097435+02:00","level":"INFO","msg":"stream: created new stream","id":"2yrqmv39"}
+{"time":"2026-06-25T13:55:13.339169961+02:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-25T13:55:13.339977953+02:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-25T13:55:13.340193949+02:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-25T13:55:13.340184957+02:00","level":"INFO","msg":"writer: started","stream_id":"2yrqmv39"}
+{"time":"2026-06-25T13:55:13.352353675+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.352367342+02:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

checkpoints/codi_frozen_1.5b_hidden/wandb/debug.log

@@ -0,0 +1,20 @@
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Configure stats pid to 730088
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:setup_run_log_directory():723] Logging user logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug.log
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug-internal.log
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():766] calling init triggers
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'model': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'sources': ['mbpp', 'humaneval', 'pyx'], 'cache_dir': 'data/cache/codi_train', 'n_samples': -1, 'max_seq_len': 3072, 'max_frames': -1, 'latent_steps': 1, 'epochs': 5.0, 'lr': 1e-05, 'batch_size': 1, 'grad_accum': 4, 'max_steps': 2500, 'save_steps': 500, 'alpha': 0.0, 'beta': 1.0, 'gamma': 1.0, 'kd_layers': None, 'frozen_teacher': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'kd_target': 'hidden', 'kd_temp': 2.0, 'single_anchor': False, 'ss_prob': 0.0, 'ss_ramp_frac': 0.5, '_wandb': {}}
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():814] starting backend
+2026-06-25 13:55:13,205 INFO    MainThread:730088 [wandb_init.py:init():829] sending inform_init request
+2026-06-25 13:55:13,340 INFO    MainThread:730088 [wandb_init.py:init():834] backend started and connected
+2026-06-25 13:55:13,342 INFO    MainThread:730088 [wandb_init.py:init():904] updated telemetry
+2026-06-25 13:55:13,350 INFO    MainThread:730088 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-06-25 13:55:13,353 INFO    MainThread:730088 [wandb_init.py:init():972] starting run threads in backend
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_console_start():2523] atexit reg
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-06-25 13:55:13,803 INFO    MainThread:730088 [wandb_init.py:init():1010] run started, returning control to user process
+2026-06-25 13:55:20,985 INFO    MainThread:730088 [wandb_run.py:_config_callback():1346] config_cb None None {'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-05, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': 2500, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': None, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'model_weights/codi_frozen_1.5b_hidden/runs/Jun25_13-55-13_alvis4-41', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': [], 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_8bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'project': 'huggingface', 'trackio_space_id': 'trackio', 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': 'no', 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': True}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/files/requirements.txt

@@ -0,0 +1,229 @@
+wheel==0.47.0
+triton==3.3.1
+ptyprocess==0.7.0
+kiwisolver==1.5.0
+pyzmq==27.1.0
+nvidia-cublas-cu12==12.6.4.1
+python-dateutil==2.9.0.post0
+cycler==0.12.1
+rfc3339-validator==0.1.4
+pluggy==1.6.0
+grpclib==0.4.9
+attrs==26.1.0
+urllib3==2.7.0
+cffi==2.0.0
+openai==1.86.0
+async-lru==2.3.0
+cfgv==3.5.0
+httpcore==1.0.9
+bitsandbytes==0.49.2
+iniconfig==2.3.0
+nvidia-cuda-runtime-cu12==12.6.77
+jupyterlab_server==2.28.0
+latex2sympy2_extended==1.11.0
+jupyter-events==0.12.1
+docker==7.1.0
+ipywidgets==8.1.8
+blinker==1.9.0
+cbor2==6.1.2
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cusolver-cu12==11.7.1.2
+multiprocess==0.70.16
+anyio==4.14.0
+itsdangerous==2.2.0
+matplotlib==3.11.0
+widgetsnbextension==4.0.15
+watchfiles==1.2.0
+swebench==4.1.0
+blobfile==3.2.0
+jupyter==1.1.1
+certifi==2026.6.17
+argon2-cffi==25.1.0
+argon2-cffi-bindings==25.1.0
+overrides==7.7.0
+propcache==0.5.2
+ipython==8.33.0
+nbconvert==7.17.1
+rich==15.0.0
+asttokens==3.0.1
+tokenizers==0.22.2
+synchronicity==0.12.4
+tqdm==4.68.3
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-nvtx-cu12==12.6.77
+jsonschema-specifications==2025.9.1
+jupyterlab==4.5.8
+pure_eval==0.2.3
+uri-template==1.3.0
+sentry-sdk==2.63.0
+pycryptodomex==3.23.0
+yarl==1.24.2
+h2==4.3.0
+h11==0.16.0
+requests==2.34.2
+Jinja2==3.1.6
+xformers==0.0.31.post1
+rfc3987-syntax==1.1.0
+antlr4-python3-runtime==4.11.0
+nvidia-cusparselt-cu12==0.6.3
+safetensors==0.8.0
+notebook_shim==0.2.4
+Pygments==2.20.0
+python-json-logger==4.1.0
+isoduration==20.11.0
+smmap==5.0.3
+click==8.3.3
+transformers==4.57.6
+aiohappyeyeballs==2.6.2
+regex==2026.5.9
+jsonpointer==3.1.1
+fqdn==1.5.1
+jupyter-lsp==2.3.1
+rfc3986-validator==0.1.1
+prometheus_client==0.25.0
+mistune==3.2.1
+sentencepiece==0.2.1
+types-toml==0.10.8.20260518
+wcwidth==0.8.1
+identify==2.6.19
+protobuf==6.33.6
+datasets==3.6.0
+accelerate==1.14.0
+networkx==3.6.1
+pydantic==2.13.4
+multidict==6.7.1
+chardet==7.4.3
+pytest==9.1.0
+babel==2.18.0
+pip==26.1.2
+dill==0.3.8
+referencing==0.37.0
+pre_commit==4.6.0
+numpy==2.4.6
+math-verify==0.9.0
+traitlets==5.15.1
+nvidia-curand-cu12==10.3.7.77
+websocket-client==1.9.0
+hf-xet==1.5.1
+unidiff==0.7.5
+nest-asyncio2==1.7.2
+typing_extensions==4.15.0
+platformdirs==4.10.0
+torch==2.7.1
+tblib==3.2.2
+sqlfluff==4.2.2
+modal==1.5.0
+ipykernel==7.3.0
+msgspec==0.21.1
+virtualenv==21.5.1
+json5==0.14.0
+pathlib_abc==0.5.2
+aiosignal==1.4.0
+moodist==0.5.1
+rpds-py==2026.5.1
+webencodings==0.5.1
+sniffio==1.3.1
+annotated-types==0.7.0
+python-discovery==1.4.2
+scipy==1.17.1
+GitPython==3.1.50
+defusedxml==0.7.1
+nvidia-cufile-cu12==1.11.1.6
+nvidia-nccl-cu12==2.26.2
+sympy==1.14.0
+tinycss2==1.5.1
+soupsieve==2.8.4
+ninja==1.13.0
+python-dotenv==1.2.2
+ghapi==1.0.13
+jupyter_server_terminals==0.5.4
+markdown-it-py==4.2.0
+nbclient==0.11.0
+mpmath==1.3.0
+frozenlist==1.8.0
+Werkzeug==3.1.8
+pillow==12.2.0
+executing==2.2.1
+gitdb==4.0.12
+pyparsing==3.3.2
+setuptools==82.0.1
+Flask==3.1.1
+MarkupSafe==3.0.3
+notebook==7.5.7
+jupyter_core==5.9.1
+jedi==0.20.0
+mdurl==0.1.2
+lark==1.3.1
+bleach==6.4.0
+hpack==4.1.0
+google-re2==1.1.20240702
+beautifulsoup4==4.15.0
+jsonschema==4.26.0
+psutil==7.2.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+narwhals==2.22.1
+fonttools==4.63.0
+lxml==6.1.1
+fastjsonschema==2.21.2
+tornado==6.5.7
+nodeenv==1.10.0
+typing-inspection==0.4.2
+httpx==0.28.1
+distro==1.9.0
+colorama==0.4.6
+jupyter-console==6.6.3
+jupyterlab_widgets==3.0.16
+PyYAML==6.0.3
+fsspec==2025.3.0
+hyperframe==6.1.0
+nvidia-cuda-cupti-cu12==12.6.80
+charset-normalizer==3.4.7
+wandb==0.27.2
+nbformat==5.10.4
+nvidia-cudnn-cu12==9.5.1.17
+peft==0.19.1
+pathspec==1.1.1
+matplotlib-inline==0.2.2
+huggingface_hub==0.36.2
+pyarrow==24.0.0
+nvidia-cusparse-cu12==12.5.4.2
+cwm==0.0.1
+prompt_toolkit==3.0.52
+omegaconf==2.4.0.dev4
+tiktoken==0.13.0
+contourpy==1.3.3
+comm==0.2.3
+threadpoolctl==3.6.0
+pydantic_core==2.46.4
+tenacity==9.1.4
+parso==0.8.7
+pandas==3.0.3
+webcolors==25.10.0
+idna==3.18
+terminado==0.18.1
+pandocfilters==1.5.1
+aiohttp==3.14.1
+arrow==1.4.0
+packaging==26.2
+tzdata==2026.2
+distlib==0.4.3
+Send2Trash==2.1.0
+jupyterlab_pygments==0.3.0
+decorator==5.3.1
+jupyter_server==2.20.0
+scikit-learn==1.9.0
+six==1.17.0
+diff_cover==10.3.0
+pycparser==3.0
+types-certifi==2021.10.8.3
+toml==0.10.2
+stack-data==0.6.3
+fastcore==1.13.5
+filelock==3.29.4
+xxhash==3.7.0
+pexpect==4.9.0
+jupyter_client==8.9.1
+jiter==0.15.0
+debugpy==1.8.21
+universal_pathlib==0.3.10
+joblib==1.5.3

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2026-06-24T18:22:48.199834583+02:00","level":"INFO","msg":"main: starting server","port-filename":"/local/tmp.6776479/tmp35_s83im/port-1496227.txt","pid":1496227,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-06-24T18:22:48.202262875+02:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/local/tmp.6776479/wandb-1496227-1496593-2615049025/socket","Net":"unix"}}
+{"time":"2026-06-24T18:22:48.202333183+02:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1496227}
+{"time":"2026-06-24T18:22:48.380047248+02:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-06-24T18:22:48.395783028+02:00","level":"INFO","msg":"handleInformInit: received","streamId":"a09es240","id":"1(@)"}
+{"time":"2026-06-24T18:22:48.524311438+02:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"a09es240","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.856561186+02:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.863547001+02:00","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.863588585+02:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-06-25T08:28:06.863722754+02:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/local/tmp.6776479/wandb-1496227-1496593-2615049025/socket","Net":"unix"}}
+{"time":"2026-06-25T08:28:06.863544627+02:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.865032377+02:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.877218227+02:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.877231736+02:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-06-25T08:28:06.877239245+02:00","level":"INFO","msg":"server is closed"}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-06-24T18:22:48.397974308+02:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-24T18:22:48.397994982+02:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-06-24T18:22:48.521885685+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-24T18:22:48.52191121+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-24T18:22:48.521927568+02:00","level":"INFO","msg":"stream: created new stream","id":"a09es240"}
+{"time":"2026-06-24T18:22:48.522085415+02:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-24T18:22:48.524303154+02:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-24T18:22:48.524446338+02:00","level":"INFO","msg":"writer: started","stream_id":"a09es240"}
+{"time":"2026-06-24T18:22:48.524476418+02:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-24T18:22:48.54574944+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-24T18:22:48.545764696+02:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/logs/debug.log

@@ -0,0 +1,20 @@
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_setup.py:_flush():81] Configure stats pid to 1496227
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_init.py:setup_run_log_directory():723] Logging user logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/logs/debug.log
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/logs/debug-internal.log
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_init.py:init():766] calling init triggers
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'model': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'sources': ['mbpp', 'humaneval', 'pyx'], 'cache_dir': 'data/cache/codi_train', 'n_samples': -1, 'max_seq_len': 3072, 'max_frames': -1, 'latent_steps': 1, 'epochs': 5.0, 'lr': 1e-05, 'batch_size': 1, 'grad_accum': 4, 'max_steps': 3000, 'save_steps': 500, 'alpha': 0.0, 'beta': 1.0, 'gamma': 1.0, 'kd_layers': [-1], 'frozen_teacher': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'kd_target': 'hidden', 'kd_temp': 2.0, 'single_anchor': False, 'ss_prob': 0.0, 'ss_ramp_frac': 0.5, '_wandb': {}}
+2026-06-24 18:22:47,975 INFO    MainThread:1496227 [wandb_init.py:init():814] starting backend
+2026-06-24 18:22:48,380 INFO    MainThread:1496227 [wandb_init.py:init():829] sending inform_init request
+2026-06-24 18:22:48,525 INFO    MainThread:1496227 [wandb_init.py:init():834] backend started and connected
+2026-06-24 18:22:48,530 INFO    MainThread:1496227 [wandb_init.py:init():904] updated telemetry
+2026-06-24 18:22:48,543 INFO    MainThread:1496227 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-06-24 18:22:48,547 INFO    MainThread:1496227 [wandb_init.py:init():972] starting run threads in backend
+2026-06-24 18:22:48,977 INFO    MainThread:1496227 [wandb_run.py:_console_start():2523] atexit reg
+2026-06-24 18:22:48,977 INFO    MainThread:1496227 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-06-24 18:22:48,977 INFO    MainThread:1496227 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-06-24 18:22:48,977 INFO    MainThread:1496227 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-06-24 18:22:48,983 INFO    MainThread:1496227 [wandb_init.py:init():1010] run started, returning control to user process
+2026-06-24 18:22:53,365 INFO    MainThread:1496227 [wandb_run.py:_config_callback():1346] config_cb None None {'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-05, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': 3000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': None, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'model_weights/codi_frozen_1.5b_hidden/runs/Jun24_18-22-48_alvis4-42', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': [], 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_8bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'project': 'huggingface', 'trackio_space_id': 'trackio', 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': 'no', 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': True}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260624_182247-a09es240/run-a09es240.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27653e5b0914a0b8352517b0092031193c440b85c22f6868e1ed93f405890f55
+size 7372800

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/files/requirements.txt

@@ -0,0 +1,229 @@
+wheel==0.47.0
+triton==3.3.1
+ptyprocess==0.7.0
+kiwisolver==1.5.0
+pyzmq==27.1.0
+nvidia-cublas-cu12==12.6.4.1
+python-dateutil==2.9.0.post0
+cycler==0.12.1
+rfc3339-validator==0.1.4
+pluggy==1.6.0
+grpclib==0.4.9
+attrs==26.1.0
+urllib3==2.7.0
+cffi==2.0.0
+openai==1.86.0
+async-lru==2.3.0
+cfgv==3.5.0
+httpcore==1.0.9
+bitsandbytes==0.49.2
+iniconfig==2.3.0
+nvidia-cuda-runtime-cu12==12.6.77
+jupyterlab_server==2.28.0
+latex2sympy2_extended==1.11.0
+jupyter-events==0.12.1
+docker==7.1.0
+ipywidgets==8.1.8
+blinker==1.9.0
+cbor2==6.1.2
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cusolver-cu12==11.7.1.2
+multiprocess==0.70.16
+anyio==4.14.0
+itsdangerous==2.2.0
+matplotlib==3.11.0
+widgetsnbextension==4.0.15
+watchfiles==1.2.0
+swebench==4.1.0
+blobfile==3.2.0
+jupyter==1.1.1
+certifi==2026.6.17
+argon2-cffi==25.1.0
+argon2-cffi-bindings==25.1.0
+overrides==7.7.0
+propcache==0.5.2
+ipython==8.33.0
+nbconvert==7.17.1
+rich==15.0.0
+asttokens==3.0.1
+tokenizers==0.22.2
+synchronicity==0.12.4
+tqdm==4.68.3
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-nvtx-cu12==12.6.77
+jsonschema-specifications==2025.9.1
+jupyterlab==4.5.8
+pure_eval==0.2.3
+uri-template==1.3.0
+sentry-sdk==2.63.0
+pycryptodomex==3.23.0
+yarl==1.24.2
+h2==4.3.0
+h11==0.16.0
+requests==2.34.2
+Jinja2==3.1.6
+xformers==0.0.31.post1
+rfc3987-syntax==1.1.0
+antlr4-python3-runtime==4.11.0
+nvidia-cusparselt-cu12==0.6.3
+safetensors==0.8.0
+notebook_shim==0.2.4
+Pygments==2.20.0
+python-json-logger==4.1.0
+isoduration==20.11.0
+smmap==5.0.3
+click==8.3.3
+transformers==4.57.6
+aiohappyeyeballs==2.6.2
+regex==2026.5.9
+jsonpointer==3.1.1
+fqdn==1.5.1
+jupyter-lsp==2.3.1
+rfc3986-validator==0.1.1
+prometheus_client==0.25.0
+mistune==3.2.1
+sentencepiece==0.2.1
+types-toml==0.10.8.20260518
+wcwidth==0.8.1
+identify==2.6.19
+protobuf==6.33.6
+datasets==3.6.0
+accelerate==1.14.0
+networkx==3.6.1
+pydantic==2.13.4
+multidict==6.7.1
+chardet==7.4.3
+pytest==9.1.0
+babel==2.18.0
+pip==26.1.2
+dill==0.3.8
+referencing==0.37.0
+pre_commit==4.6.0
+numpy==2.4.6
+math-verify==0.9.0
+traitlets==5.15.1
+nvidia-curand-cu12==10.3.7.77
+websocket-client==1.9.0
+hf-xet==1.5.1
+unidiff==0.7.5
+nest-asyncio2==1.7.2
+typing_extensions==4.15.0
+platformdirs==4.10.0
+torch==2.7.1
+tblib==3.2.2
+sqlfluff==4.2.2
+modal==1.5.0
+ipykernel==7.3.0
+msgspec==0.21.1
+virtualenv==21.5.1
+json5==0.14.0
+pathlib_abc==0.5.2
+aiosignal==1.4.0
+moodist==0.5.1
+rpds-py==2026.5.1
+webencodings==0.5.1
+sniffio==1.3.1
+annotated-types==0.7.0
+python-discovery==1.4.2
+scipy==1.17.1
+GitPython==3.1.50
+defusedxml==0.7.1
+nvidia-cufile-cu12==1.11.1.6
+nvidia-nccl-cu12==2.26.2
+sympy==1.14.0
+tinycss2==1.5.1
+soupsieve==2.8.4
+ninja==1.13.0
+python-dotenv==1.2.2
+ghapi==1.0.13
+jupyter_server_terminals==0.5.4
+markdown-it-py==4.2.0
+nbclient==0.11.0
+mpmath==1.3.0
+frozenlist==1.8.0
+Werkzeug==3.1.8
+pillow==12.2.0
+executing==2.2.1
+gitdb==4.0.12
+pyparsing==3.3.2
+setuptools==82.0.1
+Flask==3.1.1
+MarkupSafe==3.0.3
+notebook==7.5.7
+jupyter_core==5.9.1
+jedi==0.20.0
+mdurl==0.1.2
+lark==1.3.1
+bleach==6.4.0
+hpack==4.1.0
+google-re2==1.1.20240702
+beautifulsoup4==4.15.0
+jsonschema==4.26.0
+psutil==7.2.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+narwhals==2.22.1
+fonttools==4.63.0
+lxml==6.1.1
+fastjsonschema==2.21.2
+tornado==6.5.7
+nodeenv==1.10.0
+typing-inspection==0.4.2
+httpx==0.28.1
+distro==1.9.0
+colorama==0.4.6
+jupyter-console==6.6.3
+jupyterlab_widgets==3.0.16
+PyYAML==6.0.3
+fsspec==2025.3.0
+hyperframe==6.1.0
+nvidia-cuda-cupti-cu12==12.6.80
+charset-normalizer==3.4.7
+wandb==0.27.2
+nbformat==5.10.4
+nvidia-cudnn-cu12==9.5.1.17
+peft==0.19.1
+pathspec==1.1.1
+matplotlib-inline==0.2.2
+huggingface_hub==0.36.2
+pyarrow==24.0.0
+nvidia-cusparse-cu12==12.5.4.2
+cwm==0.0.1
+prompt_toolkit==3.0.52
+omegaconf==2.4.0.dev4
+tiktoken==0.13.0
+contourpy==1.3.3
+comm==0.2.3
+threadpoolctl==3.6.0
+pydantic_core==2.46.4
+tenacity==9.1.4
+parso==0.8.7
+pandas==3.0.3
+webcolors==25.10.0
+idna==3.18
+terminado==0.18.1
+pandocfilters==1.5.1
+aiohttp==3.14.1
+arrow==1.4.0
+packaging==26.2
+tzdata==2026.2
+distlib==0.4.3
+Send2Trash==2.1.0
+jupyterlab_pygments==0.3.0
+decorator==5.3.1
+jupyter_server==2.20.0
+scikit-learn==1.9.0
+six==1.17.0
+diff_cover==10.3.0
+pycparser==3.0
+types-certifi==2021.10.8.3
+toml==0.10.2
+stack-data==0.6.3
+fastcore==1.13.5
+filelock==3.29.4
+xxhash==3.7.0
+pexpect==4.9.0
+jupyter_client==8.9.1
+jiter==0.15.0
+debugpy==1.8.21
+universal_pathlib==0.3.10
+joblib==1.5.3

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug-core.log

@@ -0,0 +1,6 @@
+{"time":"2026-06-25T13:55:13.026700633+02:00","level":"INFO","msg":"main: starting server","port-filename":"/local/tmp.6777512/tmpd3vdwb53/port-730088.txt","pid":730088,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-06-25T13:55:13.027211081+02:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/local/tmp.6777512/wandb-730088-730527-3248140470/socket","Net":"unix"}}
+{"time":"2026-06-25T13:55:13.027295469+02:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":730088}
+{"time":"2026-06-25T13:55:13.205856601+02:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-06-25T13:55:13.215924274+02:00","level":"INFO","msg":"handleInformInit: received","streamId":"2yrqmv39","id":"1(@)"}
+{"time":"2026-06-25T13:55:13.339984312+02:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"2yrqmv39","id":"1(@)"}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-06-25T13:55:13.216983388+02:00","level":"INFO","msg":"wandb-core"}
+{"time":"2026-06-25T13:55:13.216997191+02:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"}
+{"time":"2026-06-25T13:55:13.339058881+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.339080395+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.339097435+02:00","level":"INFO","msg":"stream: created new stream","id":"2yrqmv39"}
+{"time":"2026-06-25T13:55:13.339169961+02:00","level":"INFO","msg":"handler: started"}
+{"time":"2026-06-25T13:55:13.339977953+02:00","level":"INFO","msg":"stream: started"}
+{"time":"2026-06-25T13:55:13.340193949+02:00","level":"INFO","msg":"sender: started"}
+{"time":"2026-06-25T13:55:13.340184957+02:00","level":"INFO","msg":"writer: started","stream_id":"2yrqmv39"}
+{"time":"2026-06-25T13:55:13.352353675+02:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-06-25T13:55:13.352367342+02:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}

checkpoints/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug.log

@@ -0,0 +1,20 @@
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Configure stats pid to 730088
+2026-06-25 13:55:12,781 INFO    MainThread:730088 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:setup_run_log_directory():723] Logging user logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug.log
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to model_weights/codi_frozen_1.5b_hidden/wandb/offline-run-20260625_135512-2yrqmv39/logs/debug-internal.log
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():766] calling init triggers
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'model': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'sources': ['mbpp', 'humaneval', 'pyx'], 'cache_dir': 'data/cache/codi_train', 'n_samples': -1, 'max_seq_len': 3072, 'max_frames': -1, 'latent_steps': 1, 'epochs': 5.0, 'lr': 1e-05, 'batch_size': 1, 'grad_accum': 4, 'max_steps': 2500, 'save_steps': 500, 'alpha': 0.0, 'beta': 1.0, 'gamma': 1.0, 'kd_layers': None, 'frozen_teacher': 'model_weights/sft1.5b_lr1e5_bs32/checkpoint-6936', 'kd_target': 'hidden', 'kd_temp': 2.0, 'single_anchor': False, 'ss_prob': 0.0, 'ss_ramp_frac': 0.5, '_wandb': {}}
+2026-06-25 13:55:12,782 INFO    MainThread:730088 [wandb_init.py:init():814] starting backend
+2026-06-25 13:55:13,205 INFO    MainThread:730088 [wandb_init.py:init():829] sending inform_init request
+2026-06-25 13:55:13,340 INFO    MainThread:730088 [wandb_init.py:init():834] backend started and connected
+2026-06-25 13:55:13,342 INFO    MainThread:730088 [wandb_init.py:init():904] updated telemetry
+2026-06-25 13:55:13,350 INFO    MainThread:730088 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-06-25 13:55:13,353 INFO    MainThread:730088 [wandb_init.py:init():972] starting run threads in backend
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_console_start():2523] atexit reg
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-06-25 13:55:13,797 INFO    MainThread:730088 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-06-25 13:55:13,803 INFO    MainThread:730088 [wandb_init.py:init():1010] run started, returning control to user process
+2026-06-25 13:55:20,985 INFO    MainThread:730088 [wandb_run.py:_config_callback():1346] config_cb None None {'output_dir': 'model_weights/codi_frozen_1.5b_hidden', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 1e-05, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': 2500, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': None, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'model_weights/codi_frozen_1.5b_hidden/runs/Jun25_13-55-13_alvis4-41', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': None, 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': [], 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'paged_adamw_8bit', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'project': 'huggingface', 'trackio_space_id': 'trackio', 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_always_push': False, 'hub_revision': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': 'no', 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'liger_kernel_config': None, 'eval_use_gather_object': False, 'average_tokens_across_devices': True}