Phase-1 LR=1e-5 Jacobi Forcing Coder checkpoint at step 5000 (normalized)

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,58 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.53.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b59ac4f41f2776f6a6930b1781789ab92d3a8119900206f232ef6546bd3df3aa
+size 4877660776

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4b52c46ae0e71f602500e86c9f3eb9152c204fa2b86fc39c906f75c57cb57fe
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2072ba1adefba819ef9f1796d282f1d8e517021ae8521bff55ef79c8a4fd3270
+size 4330865200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:235e66990ab43ca9cfc0958870adb17cdb0b02987fb836a6f683e647100b4f6b
+size 1089994880

model.safetensors.index.json

@@ -0,0 +1,347 @@
+{
+  "metadata": {
+    "total_parameters": 7615616512,
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors",
+    "lm_head.weight": "model-00004-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,3534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001,
+      "grad_norm": 43.0,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 7.7331,
+      "step": 10
+    },
+    {
+      "epoch": 0.002,
+      "grad_norm": 30.625,
+      "learning_rate": 2.1111111111111113e-07,
+      "loss": 7.9552,
+      "step": 20
+    },
+    {
+      "epoch": 0.003,
+      "grad_norm": 42.75,
+      "learning_rate": 3.2222222222222227e-07,
+      "loss": 8.0874,
+      "step": 30
+    },
+    {
+      "epoch": 0.004,
+      "grad_norm": 36.75,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 7.5538,
+      "step": 40
+    },
+    {
+      "epoch": 0.005,
+      "grad_norm": 38.5,
+      "learning_rate": 5.444444444444444e-07,
+      "loss": 8.5221,
+      "step": 50
+    },
+    {
+      "epoch": 0.006,
+      "grad_norm": 42.0,
+      "learning_rate": 6.555555555555556e-07,
+      "loss": 8.0865,
+      "step": 60
+    },
+    {
+      "epoch": 0.007,
+      "grad_norm": 51.0,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 8.0441,
+      "step": 70
+    },
+    {
+      "epoch": 0.008,
+      "grad_norm": 41.25,
+      "learning_rate": 8.777777777777778e-07,
+      "loss": 7.8363,
+      "step": 80
+    },
+    {
+      "epoch": 0.009,
+      "grad_norm": 47.75,
+      "learning_rate": 9.88888888888889e-07,
+      "loss": 8.0084,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 42.5,
+      "learning_rate": 1.1e-06,
+      "loss": 7.6296,
+      "step": 100
+    },
+    {
+      "epoch": 0.011,
+      "grad_norm": 33.75,
+      "learning_rate": 1.2111111111111111e-06,
+      "loss": 7.2967,
+      "step": 110
+    },
+    {
+      "epoch": 0.012,
+      "grad_norm": 35.25,
+      "learning_rate": 1.3222222222222222e-06,
+      "loss": 7.1319,
+      "step": 120
+    },
+    {
+      "epoch": 0.013,
+      "grad_norm": 50.25,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 7.5432,
+      "step": 130
+    },
+    {
+      "epoch": 0.014,
+      "grad_norm": 39.75,
+      "learning_rate": 1.5444444444444446e-06,
+      "loss": 7.4151,
+      "step": 140
+    },
+    {
+      "epoch": 0.015,
+      "grad_norm": 27.75,
+      "learning_rate": 1.6555555555555559e-06,
+      "loss": 6.7109,
+      "step": 150
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 35.75,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 5.9371,
+      "step": 160
+    },
+    {
+      "epoch": 0.017,
+      "grad_norm": 30.625,
+      "learning_rate": 1.8777777777777778e-06,
+      "loss": 6.5138,
+      "step": 170
+    },
+    {
+      "epoch": 0.018,
+      "grad_norm": 40.5,
+      "learning_rate": 1.988888888888889e-06,
+      "loss": 6.038,
+      "step": 180
+    },
+    {
+      "epoch": 0.019,
+      "grad_norm": 31.875,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 6.0491,
+      "step": 190
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 30.125,
+      "learning_rate": 2.2111111111111113e-06,
+      "loss": 5.8265,
+      "step": 200
+    },
+    {
+      "epoch": 0.021,
+      "grad_norm": 34.25,
+      "learning_rate": 2.3222222222222224e-06,
+      "loss": 5.4501,
+      "step": 210
+    },
+    {
+      "epoch": 0.022,
+      "grad_norm": 27.625,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 5.9894,
+      "step": 220
+    },
+    {
+      "epoch": 0.023,
+      "grad_norm": 26.0,
+      "learning_rate": 2.5444444444444446e-06,
+      "loss": 5.3144,
+      "step": 230
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 24.125,
+      "learning_rate": 2.6555555555555556e-06,
+      "loss": 5.4183,
+      "step": 240
+    },
+    {
+      "epoch": 0.025,
+      "grad_norm": 24.75,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 5.2284,
+      "step": 250
+    },
+    {
+      "epoch": 0.026,
+      "grad_norm": 25.25,
+      "learning_rate": 2.8777777777777782e-06,
+      "loss": 4.6897,
+      "step": 260
+    },
+    {
+      "epoch": 0.027,
+      "grad_norm": 25.75,
+      "learning_rate": 2.988888888888889e-06,
+      "loss": 5.0752,
+      "step": 270
+    },
+    {
+      "epoch": 0.028,
+      "grad_norm": 21.75,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 4.7286,
+      "step": 280
+    },
+    {
+      "epoch": 0.029,
+      "grad_norm": 26.625,
+      "learning_rate": 3.2111111111111115e-06,
+      "loss": 4.9517,
+      "step": 290
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 24.375,
+      "learning_rate": 3.322222222222222e-06,
+      "loss": 4.7127,
+      "step": 300
+    },
+    {
+      "epoch": 0.031,
+      "grad_norm": 17.625,
+      "learning_rate": 3.4333333333333336e-06,
+      "loss": 4.3605,
+      "step": 310
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 20.5,
+      "learning_rate": 3.5444444444444447e-06,
+      "loss": 4.7058,
+      "step": 320
+    },
+    {
+      "epoch": 0.033,
+      "grad_norm": 26.125,
+      "learning_rate": 3.6555555555555562e-06,
+      "loss": 4.3845,
+      "step": 330
+    },
+    {
+      "epoch": 0.034,
+      "grad_norm": 20.25,
+      "learning_rate": 3.766666666666667e-06,
+      "loss": 4.045,
+      "step": 340
+    },
+    {
+      "epoch": 0.035,
+      "grad_norm": 19.375,
+      "learning_rate": 3.877777777777778e-06,
+      "loss": 4.3713,
+      "step": 350
+    },
+    {
+      "epoch": 0.036,
+      "grad_norm": 26.875,
+      "learning_rate": 3.9888888888888895e-06,
+      "loss": 4.391,
+      "step": 360
+    },
+    {
+      "epoch": 0.037,
+      "grad_norm": 24.0,
+      "learning_rate": 4.1e-06,
+      "loss": 4.327,
+      "step": 370
+    },
+    {
+      "epoch": 0.038,
+      "grad_norm": 21.5,
+      "learning_rate": 4.211111111111112e-06,
+      "loss": 4.0787,
+      "step": 380
+    },
+    {
+      "epoch": 0.039,
+      "grad_norm": 18.0,
+      "learning_rate": 4.322222222222223e-06,
+      "loss": 3.9586,
+      "step": 390
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 28.875,
+      "learning_rate": 4.433333333333334e-06,
+      "loss": 4.112,
+      "step": 400
+    },
+    {
+      "epoch": 0.041,
+      "grad_norm": 21.375,
+      "learning_rate": 4.544444444444445e-06,
+      "loss": 3.7625,
+      "step": 410
+    },
+    {
+      "epoch": 0.042,
+      "grad_norm": 21.125,
+      "learning_rate": 4.655555555555556e-06,
+      "loss": 3.6063,
+      "step": 420
+    },
+    {
+      "epoch": 0.043,
+      "grad_norm": 19.625,
+      "learning_rate": 4.766666666666667e-06,
+      "loss": 3.8684,
+      "step": 430
+    },
+    {
+      "epoch": 0.044,
+      "grad_norm": 18.75,
+      "learning_rate": 4.877777777777778e-06,
+      "loss": 3.3533,
+      "step": 440
+    },
+    {
+      "epoch": 0.045,
+      "grad_norm": 22.5,
+      "learning_rate": 4.988888888888889e-06,
+      "loss": 3.403,
+      "step": 450
+    },
+    {
+      "epoch": 0.046,
+      "grad_norm": 20.0,
+      "learning_rate": 5.1e-06,
+      "loss": 3.3105,
+      "step": 460
+    },
+    {
+      "epoch": 0.047,
+      "grad_norm": 17.375,
+      "learning_rate": 5.211111111111111e-06,
+      "loss": 3.573,
+      "step": 470
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 21.25,
+      "learning_rate": 5.322222222222223e-06,
+      "loss": 3.3262,
+      "step": 480
+    },
+    {
+      "epoch": 0.049,
+      "grad_norm": 25.625,
+      "learning_rate": 5.4333333333333335e-06,
+      "loss": 3.1476,
+      "step": 490
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 20.875,
+      "learning_rate": 5.544444444444445e-06,
+      "loss": 3.3898,
+      "step": 500
+    },
+    {
+      "epoch": 0.051,
+      "grad_norm": 26.125,
+      "learning_rate": 5.6555555555555566e-06,
+      "loss": 3.5781,
+      "step": 510
+    },
+    {
+      "epoch": 0.052,
+      "grad_norm": 21.5,
+      "learning_rate": 5.766666666666667e-06,
+      "loss": 3.4613,
+      "step": 520
+    },
+    {
+      "epoch": 0.053,
+      "grad_norm": 20.5,
+      "learning_rate": 5.877777777777778e-06,
+      "loss": 3.1643,
+      "step": 530
+    },
+    {
+      "epoch": 0.054,
+      "grad_norm": 19.75,
+      "learning_rate": 5.98888888888889e-06,
+      "loss": 3.7729,
+      "step": 540
+    },
+    {
+      "epoch": 0.055,
+      "grad_norm": 21.875,
+      "learning_rate": 6.1e-06,
+      "loss": 3.3309,
+      "step": 550
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 20.0,
+      "learning_rate": 6.211111111111111e-06,
+      "loss": 3.4296,
+      "step": 560
+    },
+    {
+      "epoch": 0.057,
+      "grad_norm": 20.5,
+      "learning_rate": 6.322222222222223e-06,
+      "loss": 3.3118,
+      "step": 570
+    },
+    {
+      "epoch": 0.058,
+      "grad_norm": 19.375,
+      "learning_rate": 6.433333333333333e-06,
+      "loss": 3.2141,
+      "step": 580
+    },
+    {
+      "epoch": 0.059,
+      "grad_norm": 24.875,
+      "learning_rate": 6.544444444444445e-06,
+      "loss": 3.2603,
+      "step": 590
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 18.75,
+      "learning_rate": 6.655555555555556e-06,
+      "loss": 2.9433,
+      "step": 600
+    },
+    {
+      "epoch": 0.061,
+      "grad_norm": 19.625,
+      "learning_rate": 6.7666666666666665e-06,
+      "loss": 3.2694,
+      "step": 610
+    },
+    {
+      "epoch": 0.062,
+      "grad_norm": 22.375,
+      "learning_rate": 6.8777777777777785e-06,
+      "loss": 3.228,
+      "step": 620
+    },
+    {
+      "epoch": 0.063,
+      "grad_norm": 19.125,
+      "learning_rate": 6.9888888888888895e-06,
+      "loss": 3.28,
+      "step": 630
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 17.625,
+      "learning_rate": 7.100000000000001e-06,
+      "loss": 2.9169,
+      "step": 640
+    },
+    {
+      "epoch": 0.065,
+      "grad_norm": 22.5,
+      "learning_rate": 7.211111111111112e-06,
+      "loss": 3.0098,
+      "step": 650
+    },
+    {
+      "epoch": 0.066,
+      "grad_norm": 22.375,
+      "learning_rate": 7.322222222222223e-06,
+      "loss": 2.866,
+      "step": 660
+    },
+    {
+      "epoch": 0.067,
+      "grad_norm": 23.25,
+      "learning_rate": 7.433333333333334e-06,
+      "loss": 3.1667,
+      "step": 670
+    },
+    {
+      "epoch": 0.068,
+      "grad_norm": 19.5,
+      "learning_rate": 7.544444444444445e-06,
+      "loss": 3.1545,
+      "step": 680
+    },
+    {
+      "epoch": 0.069,
+      "grad_norm": 18.875,
+      "learning_rate": 7.655555555555556e-06,
+      "loss": 2.9292,
+      "step": 690
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 22.125,
+      "learning_rate": 7.766666666666666e-06,
+      "loss": 3.212,
+      "step": 700
+    },
+    {
+      "epoch": 0.071,
+      "grad_norm": 21.375,
+      "learning_rate": 7.877777777777778e-06,
+      "loss": 2.9498,
+      "step": 710
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 19.875,
+      "learning_rate": 7.98888888888889e-06,
+      "loss": 3.0016,
+      "step": 720
+    },
+    {
+      "epoch": 0.073,
+      "grad_norm": 20.625,
+      "learning_rate": 8.1e-06,
+      "loss": 3.0745,
+      "step": 730
+    },
+    {
+      "epoch": 0.074,
+      "grad_norm": 22.75,
+      "learning_rate": 8.211111111111112e-06,
+      "loss": 3.0406,
+      "step": 740
+    },
+    {
+      "epoch": 0.075,
+      "grad_norm": 21.0,
+      "learning_rate": 8.322222222222223e-06,
+      "loss": 3.2412,
+      "step": 750
+    },
+    {
+      "epoch": 0.076,
+      "grad_norm": 19.125,
+      "learning_rate": 8.433333333333334e-06,
+      "loss": 2.8193,
+      "step": 760
+    },
+    {
+      "epoch": 0.077,
+      "grad_norm": 20.625,
+      "learning_rate": 8.544444444444445e-06,
+      "loss": 3.1971,
+      "step": 770
+    },
+    {
+      "epoch": 0.078,
+      "grad_norm": 22.375,
+      "learning_rate": 8.655555555555557e-06,
+      "loss": 2.8541,
+      "step": 780
+    },
+    {
+      "epoch": 0.079,
+      "grad_norm": 18.0,
+      "learning_rate": 8.766666666666669e-06,
+      "loss": 2.7302,
+      "step": 790
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 20.375,
+      "learning_rate": 8.877777777777779e-06,
+      "loss": 2.6119,
+      "step": 800
+    },
+    {
+      "epoch": 0.081,
+      "grad_norm": 25.375,
+      "learning_rate": 8.988888888888889e-06,
+      "loss": 3.0592,
+      "step": 810
+    },
+    {
+      "epoch": 0.082,
+      "grad_norm": 19.625,
+      "learning_rate": 9.100000000000001e-06,
+      "loss": 2.9597,
+      "step": 820
+    },
+    {
+      "epoch": 0.083,
+      "grad_norm": 19.875,
+      "learning_rate": 9.211111111111111e-06,
+      "loss": 2.8639,
+      "step": 830
+    },
+    {
+      "epoch": 0.084,
+      "grad_norm": 22.75,
+      "learning_rate": 9.322222222222223e-06,
+      "loss": 2.9534,
+      "step": 840
+    },
+    {
+      "epoch": 0.085,
+      "grad_norm": 20.0,
+      "learning_rate": 9.433333333333335e-06,
+      "loss": 2.7801,
+      "step": 850
+    },
+    {
+      "epoch": 0.086,
+      "grad_norm": 24.375,
+      "learning_rate": 9.544444444444445e-06,
+      "loss": 2.8712,
+      "step": 860
+    },
+    {
+      "epoch": 0.087,
+      "grad_norm": 16.75,
+      "learning_rate": 9.655555555555556e-06,
+      "loss": 2.7678,
+      "step": 870
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 18.25,
+      "learning_rate": 9.766666666666667e-06,
+      "loss": 2.7104,
+      "step": 880
+    },
+    {
+      "epoch": 0.089,
+      "grad_norm": 17.875,
+      "learning_rate": 9.877777777777778e-06,
+      "loss": 3.0698,
+      "step": 890
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 20.125,
+      "learning_rate": 9.98888888888889e-06,
+      "loss": 2.7498,
+      "step": 900
+    },
+    {
+      "epoch": 0.091,
+      "grad_norm": 18.75,
+      "learning_rate": 9.999997639854591e-06,
+      "loss": 2.9519,
+      "step": 910
+    },
+    {
+      "epoch": 0.092,
+      "grad_norm": 18.0,
+      "learning_rate": 9.999989481330112e-06,
+      "loss": 2.4722,
+      "step": 920
+    },
+    {
+      "epoch": 0.093,
+      "grad_norm": 17.5,
+      "learning_rate": 9.99997549529847e-06,
+      "loss": 3.0277,
+      "step": 930
+    },
+    {
+      "epoch": 0.094,
+      "grad_norm": 19.125,
+      "learning_rate": 9.999955681775966e-06,
+      "loss": 2.7975,
+      "step": 940
+    },
+    {
+      "epoch": 0.095,
+      "grad_norm": 22.25,
+      "learning_rate": 9.999930040785695e-06,
+      "loss": 2.7893,
+      "step": 950
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 19.0,
+      "learning_rate": 9.99989857235754e-06,
+      "loss": 2.7635,
+      "step": 960
+    },
+    {
+      "epoch": 0.097,
+      "grad_norm": 21.625,
+      "learning_rate": 9.999861276528177e-06,
+      "loss": 2.5933,
+      "step": 970
+    },
+    {
+      "epoch": 0.098,
+      "grad_norm": 19.875,
+      "learning_rate": 9.999818153341078e-06,
+      "loss": 2.8434,
+      "step": 980
+    },
+    {
+      "epoch": 0.099,
+      "grad_norm": 14.6875,
+      "learning_rate": 9.999769202846499e-06,
+      "loss": 2.9051,
+      "step": 990
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 18.0,
+      "learning_rate": 9.999714425101493e-06,
+      "loss": 2.675,
+      "step": 1000
+    },
+    {
+      "epoch": 0.101,
+      "grad_norm": 19.375,
+      "learning_rate": 9.999653820169904e-06,
+      "loss": 2.718,
+      "step": 1010
+    },
+    {
+      "epoch": 0.102,
+      "grad_norm": 18.625,
+      "learning_rate": 9.99958738812237e-06,
+      "loss": 2.9848,
+      "step": 1020
+    },
+    {
+      "epoch": 0.103,
+      "grad_norm": 20.125,
+      "learning_rate": 9.999515129036312e-06,
+      "loss": 3.0125,
+      "step": 1030
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 18.25,
+      "learning_rate": 9.999437042995954e-06,
+      "loss": 2.7403,
+      "step": 1040
+    },
+    {
+      "epoch": 0.105,
+      "grad_norm": 23.25,
+      "learning_rate": 9.999353130092302e-06,
+      "loss": 2.8504,
+      "step": 1050
+    },
+    {
+      "epoch": 0.106,
+      "grad_norm": 18.875,
+      "learning_rate": 9.999263390423157e-06,
+      "loss": 2.7261,
+      "step": 1060
+    },
+    {
+      "epoch": 0.107,
+      "grad_norm": 19.875,
+      "learning_rate": 9.999167824093114e-06,
+      "loss": 2.8094,
+      "step": 1070
+    },
+    {
+      "epoch": 0.108,
+      "grad_norm": 21.25,
+      "learning_rate": 9.999066431213553e-06,
+      "loss": 2.7743,
+      "step": 1080
+    },
+    {
+      "epoch": 0.109,
+      "grad_norm": 21.125,
+      "learning_rate": 9.998959211902647e-06,
+      "loss": 2.7938,
+      "step": 1090
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 19.25,
+      "learning_rate": 9.998846166285365e-06,
+      "loss": 2.8453,
+      "step": 1100
+    },
+    {
+      "epoch": 0.111,
+      "grad_norm": 15.125,
+      "learning_rate": 9.998727294493457e-06,
+      "loss": 2.5622,
+      "step": 1110
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 17.75,
+      "learning_rate": 9.998602596665471e-06,
+      "loss": 2.4045,
+      "step": 1120
+    },
+    {
+      "epoch": 0.113,
+      "grad_norm": 20.625,
+      "learning_rate": 9.998472072946746e-06,
+      "loss": 2.9039,
+      "step": 1130
+    },
+    {
+      "epoch": 0.114,
+      "grad_norm": 18.375,
+      "learning_rate": 9.9983357234894e-06,
+      "loss": 2.5652,
+      "step": 1140
+    },
+    {
+      "epoch": 0.115,
+      "grad_norm": 18.625,
+      "learning_rate": 9.998193548452356e-06,
+      "loss": 2.6909,
+      "step": 1150
+    },
+    {
+      "epoch": 0.116,
+      "grad_norm": 18.625,
+      "learning_rate": 9.998045548001316e-06,
+      "loss": 2.8375,
+      "step": 1160
+    },
+    {
+      "epoch": 0.117,
+      "grad_norm": 21.875,
+      "learning_rate": 9.997891722308776e-06,
+      "loss": 2.8469,
+      "step": 1170
+    },
+    {
+      "epoch": 0.118,
+      "grad_norm": 17.0,
+      "learning_rate": 9.99773207155402e-06,
+      "loss": 2.9496,
+      "step": 1180
+    },
+    {
+      "epoch": 0.119,
+      "grad_norm": 20.875,
+      "learning_rate": 9.997566595923123e-06,
+      "loss": 2.7453,
+      "step": 1190
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 17.75,
+      "learning_rate": 9.997395295608945e-06,
+      "loss": 2.6202,
+      "step": 1200
+    },
+    {
+      "epoch": 0.121,
+      "grad_norm": 21.0,
+      "learning_rate": 9.99721817081114e-06,
+      "loss": 2.9364,
+      "step": 1210
+    },
+    {
+      "epoch": 0.122,
+      "grad_norm": 15.9375,
+      "learning_rate": 9.997035221736146e-06,
+      "loss": 2.9739,
+      "step": 1220
+    },
+    {
+      "epoch": 0.123,
+      "grad_norm": 18.0,
+      "learning_rate": 9.99684644859719e-06,
+      "loss": 2.696,
+      "step": 1230
+    },
+    {
+      "epoch": 0.124,
+      "grad_norm": 18.125,
+      "learning_rate": 9.996651851614292e-06,
+      "loss": 2.8032,
+      "step": 1240
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 19.625,
+      "learning_rate": 9.99645143101425e-06,
+      "loss": 2.8369,
+      "step": 1250
+    },
+    {
+      "epoch": 0.126,
+      "grad_norm": 21.125,
+      "learning_rate": 9.996245187030657e-06,
+      "loss": 2.9879,
+      "step": 1260
+    },
+    {
+      "epoch": 0.127,
+      "grad_norm": 17.25,
+      "learning_rate": 9.996033119903894e-06,
+      "loss": 2.6743,
+      "step": 1270
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 16.125,
+      "learning_rate": 9.995815229881123e-06,
+      "loss": 2.8574,
+      "step": 1280
+    },
+    {
+      "epoch": 0.129,
+      "grad_norm": 16.75,
+      "learning_rate": 9.995591517216295e-06,
+      "loss": 2.5371,
+      "step": 1290
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 16.125,
+      "learning_rate": 9.995361982170152e-06,
+      "loss": 2.7498,
+      "step": 1300
+    },
+    {
+      "epoch": 0.131,
+      "grad_norm": 16.75,
+      "learning_rate": 9.995126625010212e-06,
+      "loss": 2.8387,
+      "step": 1310
+    },
+    {
+      "epoch": 0.132,
+      "grad_norm": 16.375,
+      "learning_rate": 9.994885446010791e-06,
+      "loss": 2.6821,
+      "step": 1320
+    },
+    {
+      "epoch": 0.133,
+      "grad_norm": 21.75,
+      "learning_rate": 9.994638445452978e-06,
+      "loss": 2.6755,
+      "step": 1330
+    },
+    {
+      "epoch": 0.134,
+      "grad_norm": 15.75,
+      "learning_rate": 9.994385623624658e-06,
+      "loss": 2.5222,
+      "step": 1340
+    },
+    {
+      "epoch": 0.135,
+      "grad_norm": 23.375,
+      "learning_rate": 9.994126980820496e-06,
+      "loss": 2.6076,
+      "step": 1350
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 23.75,
+      "learning_rate": 9.993862517341937e-06,
+      "loss": 2.4664,
+      "step": 1360
+    },
+    {
+      "epoch": 0.137,
+      "grad_norm": 18.5,
+      "learning_rate": 9.993592233497216e-06,
+      "loss": 3.0119,
+      "step": 1370
+    },
+    {
+      "epoch": 0.138,
+      "grad_norm": 18.25,
+      "learning_rate": 9.993316129601352e-06,
+      "loss": 2.8505,
+      "step": 1380
+    },
+    {
+      "epoch": 0.139,
+      "grad_norm": 15.5,
+      "learning_rate": 9.993034205976141e-06,
+      "loss": 2.7681,
+      "step": 1390
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 17.0,
+      "learning_rate": 9.992746462950171e-06,
+      "loss": 2.487,
+      "step": 1400
+    },
+    {
+      "epoch": 0.141,
+      "grad_norm": 20.875,
+      "learning_rate": 9.992452900858804e-06,
+      "loss": 2.8098,
+      "step": 1410
+    },
+    {
+      "epoch": 0.142,
+      "grad_norm": 18.75,
+      "learning_rate": 9.99215352004419e-06,
+      "loss": 2.8728,
+      "step": 1420
+    },
+    {
+      "epoch": 0.143,
+      "grad_norm": 16.5,
+      "learning_rate": 9.991848320855258e-06,
+      "loss": 2.8186,
+      "step": 1430
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 20.5,
+      "learning_rate": 9.991537303647717e-06,
+      "loss": 2.7092,
+      "step": 1440
+    },
+    {
+      "epoch": 0.145,
+      "grad_norm": 18.625,
+      "learning_rate": 9.991220468784059e-06,
+      "loss": 2.5199,
+      "step": 1450
+    },
+    {
+      "epoch": 0.146,
+      "grad_norm": 17.625,
+      "learning_rate": 9.99089781663356e-06,
+      "loss": 2.5873,
+      "step": 1460
+    },
+    {
+      "epoch": 0.147,
+      "grad_norm": 20.0,
+      "learning_rate": 9.990569347572268e-06,
+      "loss": 2.4267,
+      "step": 1470
+    },
+    {
+      "epoch": 0.148,
+      "grad_norm": 19.375,
+      "learning_rate": 9.990235061983015e-06,
+      "loss": 2.8161,
+      "step": 1480
+    },
+    {
+      "epoch": 0.149,
+      "grad_norm": 15.9375,
+      "learning_rate": 9.989894960255417e-06,
+      "loss": 2.6606,
+      "step": 1490
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 14.0,
+      "learning_rate": 9.98954904278586e-06,
+      "loss": 2.6006,
+      "step": 1500
+    },
+    {
+      "epoch": 0.151,
+      "grad_norm": 20.25,
+      "learning_rate": 9.989197309977512e-06,
+      "loss": 2.8076,
+      "step": 1510
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 21.25,
+      "learning_rate": 9.98883976224032e-06,
+      "loss": 2.8272,
+      "step": 1520
+    },
+    {
+      "epoch": 0.153,
+      "grad_norm": 17.125,
+      "learning_rate": 9.988476399991006e-06,
+      "loss": 2.8416,
+      "step": 1530
+    },
+    {
+      "epoch": 0.154,
+      "grad_norm": 15.0,
+      "learning_rate": 9.988107223653072e-06,
+      "loss": 2.3451,
+      "step": 1540
+    },
+    {
+      "epoch": 0.155,
+      "grad_norm": 15.25,
+      "learning_rate": 9.987732233656793e-06,
+      "loss": 2.7907,
+      "step": 1550
+    },
+    {
+      "epoch": 0.156,
+      "grad_norm": 20.625,
+      "learning_rate": 9.987351430439223e-06,
+      "loss": 2.7175,
+      "step": 1560
+    },
+    {
+      "epoch": 0.157,
+      "grad_norm": 19.375,
+      "learning_rate": 9.986964814444189e-06,
+      "loss": 3.1835,
+      "step": 1570
+    },
+    {
+      "epoch": 0.158,
+      "grad_norm": 23.25,
+      "learning_rate": 9.98657238612229e-06,
+      "loss": 2.687,
+      "step": 1580
+    },
+    {
+      "epoch": 0.159,
+      "grad_norm": 18.5,
+      "learning_rate": 9.986174145930909e-06,
+      "loss": 2.5697,
+      "step": 1590
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 23.25,
+      "learning_rate": 9.985770094334193e-06,
+      "loss": 2.5566,
+      "step": 1600
+    },
+    {
+      "epoch": 0.161,
+      "grad_norm": 19.25,
+      "learning_rate": 9.985360231803065e-06,
+      "loss": 2.4284,
+      "step": 1610
+    },
+    {
+      "epoch": 0.162,
+      "grad_norm": 15.5625,
+      "learning_rate": 9.984944558815222e-06,
+      "loss": 2.5651,
+      "step": 1620
+    },
+    {
+      "epoch": 0.163,
+      "grad_norm": 18.5,
+      "learning_rate": 9.984523075855133e-06,
+      "loss": 2.5574,
+      "step": 1630
+    },
+    {
+      "epoch": 0.164,
+      "grad_norm": 20.25,
+      "learning_rate": 9.984095783414037e-06,
+      "loss": 2.6908,
+      "step": 1640
+    },
+    {
+      "epoch": 0.165,
+      "grad_norm": 19.5,
+      "learning_rate": 9.983662681989946e-06,
+      "loss": 2.4126,
+      "step": 1650
+    },
+    {
+      "epoch": 0.166,
+      "grad_norm": 23.5,
+      "learning_rate": 9.983223772087642e-06,
+      "loss": 2.6288,
+      "step": 1660
+    },
+    {
+      "epoch": 0.167,
+      "grad_norm": 17.75,
+      "learning_rate": 9.982779054218673e-06,
+      "loss": 2.7709,
+      "step": 1670
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 17.0,
+      "learning_rate": 9.982328528901366e-06,
+      "loss": 2.8173,
+      "step": 1680
+    },
+    {
+      "epoch": 0.169,
+      "grad_norm": 20.0,
+      "learning_rate": 9.981872196660803e-06,
+      "loss": 2.7205,
+      "step": 1690
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 21.625,
+      "learning_rate": 9.981410058028845e-06,
+      "loss": 2.6691,
+      "step": 1700
+    },
+    {
+      "epoch": 0.171,
+      "grad_norm": 20.375,
+      "learning_rate": 9.980942113544114e-06,
+      "loss": 2.6803,
+      "step": 1710
+    },
+    {
+      "epoch": 0.172,
+      "grad_norm": 21.5,
+      "learning_rate": 9.980468363752004e-06,
+      "loss": 2.8254,
+      "step": 1720
+    },
+    {
+      "epoch": 0.173,
+      "grad_norm": 20.75,
+      "learning_rate": 9.979988809204667e-06,
+      "loss": 2.8123,
+      "step": 1730
+    },
+    {
+      "epoch": 0.174,
+      "grad_norm": 20.5,
+      "learning_rate": 9.979503450461034e-06,
+      "loss": 2.6096,
+      "step": 1740
+    },
+    {
+      "epoch": 0.175,
+      "grad_norm": 21.0,
+      "learning_rate": 9.979012288086785e-06,
+      "loss": 2.8623,
+      "step": 1750
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 16.5,
+      "learning_rate": 9.978515322654374e-06,
+      "loss": 2.9909,
+      "step": 1760
+    },
+    {
+      "epoch": 0.177,
+      "grad_norm": 19.125,
+      "learning_rate": 9.978012554743017e-06,
+      "loss": 2.7251,
+      "step": 1770
+    },
+    {
+      "epoch": 0.178,
+      "grad_norm": 17.25,
+      "learning_rate": 9.977503984938692e-06,
+      "loss": 2.9922,
+      "step": 1780
+    },
+    {
+      "epoch": 0.179,
+      "grad_norm": 19.875,
+      "learning_rate": 9.976989613834138e-06,
+      "loss": 2.4104,
+      "step": 1790
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 20.0,
+      "learning_rate": 9.976469442028857e-06,
+      "loss": 2.6709,
+      "step": 1800
+    },
+    {
+      "epoch": 0.181,
+      "grad_norm": 20.875,
+      "learning_rate": 9.97594347012911e-06,
+      "loss": 2.4566,
+      "step": 1810
+    },
+    {
+      "epoch": 0.182,
+      "grad_norm": 18.375,
+      "learning_rate": 9.975411698747923e-06,
+      "loss": 2.4003,
+      "step": 1820
+    },
+    {
+      "epoch": 0.183,
+      "grad_norm": 18.375,
+      "learning_rate": 9.974874128505075e-06,
+      "loss": 2.6298,
+      "step": 1830
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 21.0,
+      "learning_rate": 9.974330760027104e-06,
+      "loss": 2.5359,
+      "step": 1840
+    },
+    {
+      "epoch": 0.185,
+      "grad_norm": 17.25,
+      "learning_rate": 9.973781593947311e-06,
+      "loss": 2.5899,
+      "step": 1850
+    },
+    {
+      "epoch": 0.186,
+      "grad_norm": 17.0,
+      "learning_rate": 9.973226630905753e-06,
+      "loss": 2.5445,
+      "step": 1860
+    },
+    {
+      "epoch": 0.187,
+      "grad_norm": 18.5,
+      "learning_rate": 9.972665871549238e-06,
+      "loss": 2.948,
+      "step": 1870
+    },
+    {
+      "epoch": 0.188,
+      "grad_norm": 19.25,
+      "learning_rate": 9.972099316531335e-06,
+      "loss": 2.1919,
+      "step": 1880
+    },
+    {
+      "epoch": 0.189,
+      "grad_norm": 16.75,
+      "learning_rate": 9.971526966512364e-06,
+      "loss": 2.6727,
+      "step": 1890
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 18.625,
+      "learning_rate": 9.970948822159402e-06,
+      "loss": 2.7471,
+      "step": 1900
+    },
+    {
+      "epoch": 0.191,
+      "grad_norm": 18.875,
+      "learning_rate": 9.970364884146282e-06,
+      "loss": 2.5303,
+      "step": 1910
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 20.5,
+      "learning_rate": 9.969775153153583e-06,
+      "loss": 2.7076,
+      "step": 1920
+    },
+    {
+      "epoch": 0.193,
+      "grad_norm": 17.375,
+      "learning_rate": 9.969179629868636e-06,
+      "loss": 2.5328,
+      "step": 1930
+    },
+    {
+      "epoch": 0.194,
+      "grad_norm": 18.25,
+      "learning_rate": 9.968578314985533e-06,
+      "loss": 2.8174,
+      "step": 1940
+    },
+    {
+      "epoch": 0.195,
+      "grad_norm": 19.75,
+      "learning_rate": 9.967971209205102e-06,
+      "loss": 2.7,
+      "step": 1950
+    },
+    {
+      "epoch": 0.196,
+      "grad_norm": 20.25,
+      "learning_rate": 9.96735831323493e-06,
+      "loss": 2.8078,
+      "step": 1960
+    },
+    {
+      "epoch": 0.197,
+      "grad_norm": 22.125,
+      "learning_rate": 9.966739627789349e-06,
+      "loss": 2.8192,
+      "step": 1970
+    },
+    {
+      "epoch": 0.198,
+      "grad_norm": 19.75,
+      "learning_rate": 9.966115153589439e-06,
+      "loss": 2.4911,
+      "step": 1980
+    },
+    {
+      "epoch": 0.199,
+      "grad_norm": 20.5,
+      "learning_rate": 9.965484891363029e-06,
+      "loss": 2.9307,
+      "step": 1990
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 19.625,
+      "learning_rate": 9.96484884184469e-06,
+      "loss": 2.5714,
+      "step": 2000
+    },
+    {
+      "epoch": 0.201,
+      "grad_norm": 18.625,
+      "learning_rate": 9.964207005775741e-06,
+      "loss": 2.5079,
+      "step": 2010
+    },
+    {
+      "epoch": 0.202,
+      "grad_norm": 16.75,
+      "learning_rate": 9.963559383904244e-06,
+      "loss": 2.6346,
+      "step": 2020
+    },
+    {
+      "epoch": 0.203,
+      "grad_norm": 19.5,
+      "learning_rate": 9.962905976985007e-06,
+      "loss": 2.8173,
+      "step": 2030
+    },
+    {
+      "epoch": 0.204,
+      "grad_norm": 18.625,
+      "learning_rate": 9.962246785779575e-06,
+      "loss": 2.5824,
+      "step": 2040
+    },
+    {
+      "epoch": 0.205,
+      "grad_norm": 21.125,
+      "learning_rate": 9.96158181105624e-06,
+      "loss": 2.7661,
+      "step": 2050
+    },
+    {
+      "epoch": 0.206,
+      "grad_norm": 20.0,
+      "learning_rate": 9.960911053590033e-06,
+      "loss": 2.7627,
+      "step": 2060
+    },
+    {
+      "epoch": 0.207,
+      "grad_norm": 20.5,
+      "learning_rate": 9.96023451416272e-06,
+      "loss": 2.8217,
+      "step": 2070
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 21.875,
+      "learning_rate": 9.959552193562816e-06,
+      "loss": 2.8213,
+      "step": 2080
+    },
+    {
+      "epoch": 0.209,
+      "grad_norm": 16.75,
+      "learning_rate": 9.958864092585567e-06,
+      "loss": 2.4847,
+      "step": 2090
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 18.25,
+      "learning_rate": 9.958170212032955e-06,
+      "loss": 2.3802,
+      "step": 2100
+    },
+    {
+      "epoch": 0.211,
+      "grad_norm": 19.375,
+      "learning_rate": 9.957470552713701e-06,
+      "loss": 2.9577,
+      "step": 2110
+    },
+    {
+      "epoch": 0.212,
+      "grad_norm": 16.75,
+      "learning_rate": 9.956765115443264e-06,
+      "loss": 2.4746,
+      "step": 2120
+    },
+    {
+      "epoch": 0.213,
+      "grad_norm": 15.375,
+      "learning_rate": 9.956053901043831e-06,
+      "loss": 2.8087,
+      "step": 2130
+    },
+    {
+      "epoch": 0.214,
+      "grad_norm": 18.25,
+      "learning_rate": 9.955336910344326e-06,
+      "loss": 2.7429,
+      "step": 2140
+    },
+    {
+      "epoch": 0.215,
+      "grad_norm": 17.75,
+      "learning_rate": 9.954614144180403e-06,
+      "loss": 2.9068,
+      "step": 2150
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 23.0,
+      "learning_rate": 9.953885603394449e-06,
+      "loss": 2.9658,
+      "step": 2160
+    },
+    {
+      "epoch": 0.217,
+      "grad_norm": 20.125,
+      "learning_rate": 9.953151288835583e-06,
+      "loss": 2.9416,
+      "step": 2170
+    },
+    {
+      "epoch": 0.218,
+      "grad_norm": 18.5,
+      "learning_rate": 9.95241120135965e-06,
+      "loss": 2.6781,
+      "step": 2180
+    },
+    {
+      "epoch": 0.219,
+      "grad_norm": 19.875,
+      "learning_rate": 9.951665341829227e-06,
+      "loss": 2.5796,
+      "step": 2190
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 23.875,
+      "learning_rate": 9.950913711113613e-06,
+      "loss": 2.7474,
+      "step": 2200
+    },
+    {
+      "epoch": 0.221,
+      "grad_norm": 20.625,
+      "learning_rate": 9.950156310088836e-06,
+      "loss": 2.671,
+      "step": 2210
+    },
+    {
+      "epoch": 0.222,
+      "grad_norm": 24.0,
+      "learning_rate": 9.949393139637655e-06,
+      "loss": 2.8648,
+      "step": 2220
+    },
+    {
+      "epoch": 0.223,
+      "grad_norm": 17.125,
+      "learning_rate": 9.948624200649544e-06,
+      "loss": 2.6385,
+      "step": 2230
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 26.25,
+      "learning_rate": 9.947849494020705e-06,
+      "loss": 2.5898,
+      "step": 2240
+    },
+    {
+      "epoch": 0.225,
+      "grad_norm": 19.625,
+      "learning_rate": 9.947069020654062e-06,
+      "loss": 2.517,
+      "step": 2250
+    },
+    {
+      "epoch": 0.226,
+      "grad_norm": 17.375,
+      "learning_rate": 9.946282781459259e-06,
+      "loss": 2.4048,
+      "step": 2260
+    },
+    {
+      "epoch": 0.227,
+      "grad_norm": 20.75,
+      "learning_rate": 9.945490777352662e-06,
+      "loss": 2.6212,
+      "step": 2270
+    },
+    {
+      "epoch": 0.228,
+      "grad_norm": 18.375,
+      "learning_rate": 9.944693009257354e-06,
+      "loss": 2.8589,
+      "step": 2280
+    },
+    {
+      "epoch": 0.229,
+      "grad_norm": 17.75,
+      "learning_rate": 9.943889478103137e-06,
+      "loss": 2.6638,
+      "step": 2290
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 18.375,
+      "learning_rate": 9.94308018482653e-06,
+      "loss": 2.4957,
+      "step": 2300
+    },
+    {
+      "epoch": 0.231,
+      "grad_norm": 15.0625,
+      "learning_rate": 9.942265130370767e-06,
+      "loss": 2.5983,
+      "step": 2310
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 17.5,
+      "learning_rate": 9.941444315685798e-06,
+      "loss": 2.4284,
+      "step": 2320
+    },
+    {
+      "epoch": 0.233,
+      "grad_norm": 20.375,
+      "learning_rate": 9.940617741728284e-06,
+      "loss": 2.7618,
+      "step": 2330
+    },
+    {
+      "epoch": 0.234,
+      "grad_norm": 19.125,
+      "learning_rate": 9.939785409461603e-06,
+      "loss": 2.7838,
+      "step": 2340
+    },
+    {
+      "epoch": 0.235,
+      "grad_norm": 18.625,
+      "learning_rate": 9.938947319855839e-06,
+      "loss": 2.3683,
+      "step": 2350
+    },
+    {
+      "epoch": 0.236,
+      "grad_norm": 19.625,
+      "learning_rate": 9.93810347388779e-06,
+      "loss": 2.715,
+      "step": 2360
+    },
+    {
+      "epoch": 0.237,
+      "grad_norm": 20.625,
+      "learning_rate": 9.93725387254096e-06,
+      "loss": 2.9065,
+      "step": 2370
+    },
+    {
+      "epoch": 0.238,
+      "grad_norm": 18.0,
+      "learning_rate": 9.936398516805565e-06,
+      "loss": 2.8114,
+      "step": 2380
+    },
+    {
+      "epoch": 0.239,
+      "grad_norm": 17.625,
+      "learning_rate": 9.935537407678522e-06,
+      "loss": 2.6193,
+      "step": 2390
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 17.75,
+      "learning_rate": 9.934670546163463e-06,
+      "loss": 2.3808,
+      "step": 2400
+    },
+    {
+      "epoch": 0.241,
+      "grad_norm": 20.375,
+      "learning_rate": 9.933797933270711e-06,
+      "loss": 2.7193,
+      "step": 2410
+    },
+    {
+      "epoch": 0.242,
+      "grad_norm": 17.5,
+      "learning_rate": 9.932919570017305e-06,
+      "loss": 2.3256,
+      "step": 2420
+    },
+    {
+      "epoch": 0.243,
+      "grad_norm": 23.75,
+      "learning_rate": 9.93203545742698e-06,
+      "loss": 2.6923,
+      "step": 2430
+    },
+    {
+      "epoch": 0.244,
+      "grad_norm": 19.375,
+      "learning_rate": 9.931145596530172e-06,
+      "loss": 2.8172,
+      "step": 2440
+    },
+    {
+      "epoch": 0.245,
+      "grad_norm": 19.25,
+      "learning_rate": 9.930249988364016e-06,
+      "loss": 2.5078,
+      "step": 2450
+    },
+    {
+      "epoch": 0.246,
+      "grad_norm": 17.625,
+      "learning_rate": 9.92934863397235e-06,
+      "loss": 2.506,
+      "step": 2460
+    },
+    {
+      "epoch": 0.247,
+      "grad_norm": 16.625,
+      "learning_rate": 9.928441534405702e-06,
+      "loss": 2.6793,
+      "step": 2470
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 23.125,
+      "learning_rate": 9.927528690721304e-06,
+      "loss": 2.8273,
+      "step": 2480
+    },
+    {
+      "epoch": 0.249,
+      "grad_norm": 23.5,
+      "learning_rate": 9.926610103983076e-06,
+      "loss": 2.7844,
+      "step": 2490
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 23.0,
+      "learning_rate": 9.925685775261636e-06,
+      "loss": 2.9647,
+      "step": 2500
+    },
+    {
+      "epoch": 0.251,
+      "grad_norm": 20.375,
+      "learning_rate": 9.924755705634292e-06,
+      "loss": 2.5624,
+      "step": 2510
+    },
+    {
+      "epoch": 0.252,
+      "grad_norm": 19.0,
+      "learning_rate": 9.923819896185043e-06,
+      "loss": 2.6843,
+      "step": 2520
+    },
+    {
+      "epoch": 0.253,
+      "grad_norm": 16.625,
+      "learning_rate": 9.92287834800458e-06,
+      "loss": 2.735,
+      "step": 2530
+    },
+    {
+      "epoch": 0.254,
+      "grad_norm": 22.125,
+      "learning_rate": 9.921931062190281e-06,
+      "loss": 2.8574,
+      "step": 2540
+    },
+    {
+      "epoch": 0.255,
+      "grad_norm": 19.75,
+      "learning_rate": 9.92097803984621e-06,
+      "loss": 2.8776,
+      "step": 2550
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 19.375,
+      "learning_rate": 9.92001928208312e-06,
+      "loss": 2.604,
+      "step": 2560
+    },
+    {
+      "epoch": 0.257,
+      "grad_norm": 19.625,
+      "learning_rate": 9.919054790018445e-06,
+      "loss": 2.8309,
+      "step": 2570
+    },
+    {
+      "epoch": 0.258,
+      "grad_norm": 15.4375,
+      "learning_rate": 9.918084564776308e-06,
+      "loss": 2.7132,
+      "step": 2580
+    },
+    {
+      "epoch": 0.259,
+      "grad_norm": 16.125,
+      "learning_rate": 9.917108607487504e-06,
+      "loss": 2.5709,
+      "step": 2590
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 24.125,
+      "learning_rate": 9.91612691928952e-06,
+      "loss": 2.6128,
+      "step": 2600
+    },
+    {
+      "epoch": 0.261,
+      "grad_norm": 20.375,
+      "learning_rate": 9.915139501326517e-06,
+      "loss": 2.3172,
+      "step": 2610
+    },
+    {
+      "epoch": 0.262,
+      "grad_norm": 18.25,
+      "learning_rate": 9.914146354749333e-06,
+      "loss": 2.5294,
+      "step": 2620
+    },
+    {
+      "epoch": 0.263,
+      "grad_norm": 18.375,
+      "learning_rate": 9.913147480715482e-06,
+      "loss": 2.4213,
+      "step": 2630
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 21.125,
+      "learning_rate": 9.91214288038916e-06,
+      "loss": 2.7051,
+      "step": 2640
+    },
+    {
+      "epoch": 0.265,
+      "grad_norm": 20.75,
+      "learning_rate": 9.911132554941231e-06,
+      "loss": 2.8351,
+      "step": 2650
+    },
+    {
+      "epoch": 0.266,
+      "grad_norm": 17.0,
+      "learning_rate": 9.910116505549233e-06,
+      "loss": 2.5062,
+      "step": 2660
+    },
+    {
+      "epoch": 0.267,
+      "grad_norm": 18.625,
+      "learning_rate": 9.909094733397375e-06,
+      "loss": 2.4828,
+      "step": 2670
+    },
+    {
+      "epoch": 0.268,
+      "grad_norm": 17.75,
+      "learning_rate": 9.908067239676538e-06,
+      "loss": 2.5945,
+      "step": 2680
+    },
+    {
+      "epoch": 0.269,
+      "grad_norm": 19.0,
+      "learning_rate": 9.907034025584267e-06,
+      "loss": 2.5403,
+      "step": 2690
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 21.75,
+      "learning_rate": 9.905995092324781e-06,
+      "loss": 2.4999,
+      "step": 2700
+    },
+    {
+      "epoch": 0.271,
+      "grad_norm": 22.125,
+      "learning_rate": 9.904950441108957e-06,
+      "loss": 2.7494,
+      "step": 2710
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 17.25,
+      "learning_rate": 9.903900073154343e-06,
+      "loss": 2.5713,
+      "step": 2720
+    },
+    {
+      "epoch": 0.273,
+      "grad_norm": 18.75,
+      "learning_rate": 9.902843989685144e-06,
+      "loss": 2.5711,
+      "step": 2730
+    },
+    {
+      "epoch": 0.274,
+      "grad_norm": 17.875,
+      "learning_rate": 9.901782191932233e-06,
+      "loss": 2.7827,
+      "step": 2740
+    },
+    {
+      "epoch": 0.275,
+      "grad_norm": 20.875,
+      "learning_rate": 9.900714681133135e-06,
+      "loss": 2.5141,
+      "step": 2750
+    },
+    {
+      "epoch": 0.276,
+      "grad_norm": 21.375,
+      "learning_rate": 9.899641458532041e-06,
+      "loss": 2.4677,
+      "step": 2760
+    },
+    {
+      "epoch": 0.277,
+      "grad_norm": 18.25,
+      "learning_rate": 9.898562525379797e-06,
+      "loss": 2.5868,
+      "step": 2770
+    },
+    {
+      "epoch": 0.278,
+      "grad_norm": 20.0,
+      "learning_rate": 9.897477882933902e-06,
+      "loss": 2.5782,
+      "step": 2780
+    },
+    {
+      "epoch": 0.279,
+      "grad_norm": 19.875,
+      "learning_rate": 9.896387532458509e-06,
+      "loss": 2.6296,
+      "step": 2790
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 21.125,
+      "learning_rate": 9.89529147522443e-06,
+      "loss": 2.7367,
+      "step": 2800
+    },
+    {
+      "epoch": 0.281,
+      "grad_norm": 19.625,
+      "learning_rate": 9.894189712509122e-06,
+      "loss": 2.8607,
+      "step": 2810
+    },
+    {
+      "epoch": 0.282,
+      "grad_norm": 18.125,
+      "learning_rate": 9.893082245596692e-06,
+      "loss": 2.6773,
+      "step": 2820
+    },
+    {
+      "epoch": 0.283,
+      "grad_norm": 17.0,
+      "learning_rate": 9.891969075777902e-06,
+      "loss": 2.6371,
+      "step": 2830
+    },
+    {
+      "epoch": 0.284,
+      "grad_norm": 17.375,
+      "learning_rate": 9.890850204350148e-06,
+      "loss": 2.6154,
+      "step": 2840
+    },
+    {
+      "epoch": 0.285,
+      "grad_norm": 20.25,
+      "learning_rate": 9.889725632617486e-06,
+      "loss": 2.3764,
+      "step": 2850
+    },
+    {
+      "epoch": 0.286,
+      "grad_norm": 23.0,
+      "learning_rate": 9.888595361890605e-06,
+      "loss": 2.9898,
+      "step": 2860
+    },
+    {
+      "epoch": 0.287,
+      "grad_norm": 20.75,
+      "learning_rate": 9.887459393486843e-06,
+      "loss": 2.7216,
+      "step": 2870
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 20.75,
+      "learning_rate": 9.886317728730172e-06,
+      "loss": 2.7847,
+      "step": 2880
+    },
+    {
+      "epoch": 0.289,
+      "grad_norm": 18.0,
+      "learning_rate": 9.885170368951209e-06,
+      "loss": 2.6752,
+      "step": 2890
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 31.25,
+      "learning_rate": 9.884017315487205e-06,
+      "loss": 2.6616,
+      "step": 2900
+    },
+    {
+      "epoch": 0.291,
+      "grad_norm": 17.75,
+      "learning_rate": 9.882858569682048e-06,
+      "loss": 2.623,
+      "step": 2910
+    },
+    {
+      "epoch": 0.292,
+      "grad_norm": 21.5,
+      "learning_rate": 9.881694132886263e-06,
+      "loss": 2.4582,
+      "step": 2920
+    },
+    {
+      "epoch": 0.293,
+      "grad_norm": 19.5,
+      "learning_rate": 9.880524006457004e-06,
+      "loss": 2.7289,
+      "step": 2930
+    },
+    {
+      "epoch": 0.294,
+      "grad_norm": 16.625,
+      "learning_rate": 9.879348191758058e-06,
+      "loss": 2.5351,
+      "step": 2940
+    },
+    {
+      "epoch": 0.295,
+      "grad_norm": 19.75,
+      "learning_rate": 9.878166690159843e-06,
+      "loss": 2.405,
+      "step": 2950
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 15.5,
+      "learning_rate": 9.8769795030394e-06,
+      "loss": 2.8448,
+      "step": 2960
+    },
+    {
+      "epoch": 0.297,
+      "grad_norm": 19.125,
+      "learning_rate": 9.875786631780404e-06,
+      "loss": 2.6537,
+      "step": 2970
+    },
+    {
+      "epoch": 0.298,
+      "grad_norm": 21.125,
+      "learning_rate": 9.87458807777315e-06,
+      "loss": 2.9104,
+      "step": 2980
+    },
+    {
+      "epoch": 0.299,
+      "grad_norm": 19.125,
+      "learning_rate": 9.873383842414557e-06,
+      "loss": 2.5027,
+      "step": 2990
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 16.375,
+      "learning_rate": 9.872173927108165e-06,
+      "loss": 2.565,
+      "step": 3000
+    },
+    {
+      "epoch": 0.301,
+      "grad_norm": 20.375,
+      "learning_rate": 9.870958333264137e-06,
+      "loss": 2.9015,
+      "step": 3010
+    },
+    {
+      "epoch": 0.302,
+      "grad_norm": 22.125,
+      "learning_rate": 9.869737062299252e-06,
+      "loss": 2.6939,
+      "step": 3020
+    },
+    {
+      "epoch": 0.303,
+      "grad_norm": 18.375,
+      "learning_rate": 9.868510115636905e-06,
+      "loss": 2.6959,
+      "step": 3030
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 20.5,
+      "learning_rate": 9.867277494707106e-06,
+      "loss": 2.5169,
+      "step": 3040
+    },
+    {
+      "epoch": 0.305,
+      "grad_norm": 18.0,
+      "learning_rate": 9.866039200946483e-06,
+      "loss": 2.7834,
+      "step": 3050
+    },
+    {
+      "epoch": 0.306,
+      "grad_norm": 14.125,
+      "learning_rate": 9.864795235798267e-06,
+      "loss": 2.7562,
+      "step": 3060
+    },
+    {
+      "epoch": 0.307,
+      "grad_norm": 16.25,
+      "learning_rate": 9.863545600712311e-06,
+      "loss": 2.3539,
+      "step": 3070
+    },
+    {
+      "epoch": 0.308,
+      "grad_norm": 18.375,
+      "learning_rate": 9.862290297145065e-06,
+      "loss": 2.7669,
+      "step": 3080
+    },
+    {
+      "epoch": 0.309,
+      "grad_norm": 20.25,
+      "learning_rate": 9.86102932655959e-06,
+      "loss": 2.897,
+      "step": 3090
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 19.125,
+      "learning_rate": 9.859762690425555e-06,
+      "loss": 2.5958,
+      "step": 3100
+    },
+    {
+      "epoch": 0.311,
+      "grad_norm": 18.75,
+      "learning_rate": 9.858490390219227e-06,
+      "loss": 2.9046,
+      "step": 3110
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 21.0,
+      "learning_rate": 9.857212427423478e-06,
+      "loss": 2.4583,
+      "step": 3120
+    },
+    {
+      "epoch": 0.313,
+      "grad_norm": 22.375,
+      "learning_rate": 9.855928803527776e-06,
+      "loss": 2.9481,
+      "step": 3130
+    },
+    {
+      "epoch": 0.314,
+      "grad_norm": 19.125,
+      "learning_rate": 9.854639520028196e-06,
+      "loss": 2.9641,
+      "step": 3140
+    },
+    {
+      "epoch": 0.315,
+      "grad_norm": 15.8125,
+      "learning_rate": 9.853344578427393e-06,
+      "loss": 2.8134,
+      "step": 3150
+    },
+    {
+      "epoch": 0.316,
+      "grad_norm": 18.875,
+      "learning_rate": 9.852043980234635e-06,
+      "loss": 2.5857,
+      "step": 3160
+    },
+    {
+      "epoch": 0.317,
+      "grad_norm": 22.875,
+      "learning_rate": 9.850737726965773e-06,
+      "loss": 2.5806,
+      "step": 3170
+    },
+    {
+      "epoch": 0.318,
+      "grad_norm": 20.625,
+      "learning_rate": 9.849425820143245e-06,
+      "loss": 3.1725,
+      "step": 3180
+    },
+    {
+      "epoch": 0.319,
+      "grad_norm": 16.75,
+      "learning_rate": 9.84810826129609e-06,
+      "loss": 2.6841,
+      "step": 3190
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 17.125,
+      "learning_rate": 9.846785051959924e-06,
+      "loss": 2.5288,
+      "step": 3200
+    },
+    {
+      "epoch": 0.321,
+      "grad_norm": 20.375,
+      "learning_rate": 9.845456193676952e-06,
+      "loss": 2.488,
+      "step": 3210
+    },
+    {
+      "epoch": 0.322,
+      "grad_norm": 18.5,
+      "learning_rate": 9.844121687995968e-06,
+      "loss": 2.4729,
+      "step": 3220
+    },
+    {
+      "epoch": 0.323,
+      "grad_norm": 17.25,
+      "learning_rate": 9.842781536472339e-06,
+      "loss": 2.5037,
+      "step": 3230
+    },
+    {
+      "epoch": 0.324,
+      "grad_norm": 19.25,
+      "learning_rate": 9.841435740668018e-06,
+      "loss": 2.649,
+      "step": 3240
+    },
+    {
+      "epoch": 0.325,
+      "grad_norm": 17.375,
+      "learning_rate": 9.840084302151537e-06,
+      "loss": 2.4221,
+      "step": 3250
+    },
+    {
+      "epoch": 0.326,
+      "grad_norm": 18.375,
+      "learning_rate": 9.838727222498001e-06,
+      "loss": 2.5987,
+      "step": 3260
+    },
+    {
+      "epoch": 0.327,
+      "grad_norm": 15.9375,
+      "learning_rate": 9.837364503289093e-06,
+      "loss": 2.3907,
+      "step": 3270
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 19.25,
+      "learning_rate": 9.835996146113064e-06,
+      "loss": 2.7607,
+      "step": 3280
+    },
+    {
+      "epoch": 0.329,
+      "grad_norm": 19.5,
+      "learning_rate": 9.834622152564746e-06,
+      "loss": 2.7398,
+      "step": 3290
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 17.25,
+      "learning_rate": 9.833242524245528e-06,
+      "loss": 2.5979,
+      "step": 3300
+    },
+    {
+      "epoch": 0.331,
+      "grad_norm": 16.5,
+      "learning_rate": 9.831857262763375e-06,
+      "loss": 2.4874,
+      "step": 3310
+    },
+    {
+      "epoch": 0.332,
+      "grad_norm": 20.125,
+      "learning_rate": 9.830466369732814e-06,
+      "loss": 2.7418,
+      "step": 3320
+    },
+    {
+      "epoch": 0.333,
+      "grad_norm": 20.875,
+      "learning_rate": 9.829069846774935e-06,
+      "loss": 2.5257,
+      "step": 3330
+    },
+    {
+      "epoch": 0.334,
+      "grad_norm": 19.0,
+      "learning_rate": 9.827667695517393e-06,
+      "loss": 2.5789,
+      "step": 3340
+    },
+    {
+      "epoch": 0.335,
+      "grad_norm": 18.75,
+      "learning_rate": 9.8262599175944e-06,
+      "loss": 2.5456,
+      "step": 3350
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 22.75,
+      "learning_rate": 9.824846514646726e-06,
+      "loss": 2.9754,
+      "step": 3360
+    },
+    {
+      "epoch": 0.337,
+      "grad_norm": 18.625,
+      "learning_rate": 9.8234274883217e-06,
+      "loss": 2.5788,
+      "step": 3370
+    },
+    {
+      "epoch": 0.338,
+      "grad_norm": 18.875,
+      "learning_rate": 9.8220028402732e-06,
+      "loss": 2.6314,
+      "step": 3380
+    },
+    {
+      "epoch": 0.339,
+      "grad_norm": 19.125,
+      "learning_rate": 9.820572572161658e-06,
+      "loss": 2.508,
+      "step": 3390
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 19.625,
+      "learning_rate": 9.819136685654059e-06,
+      "loss": 2.9974,
+      "step": 3400
+    },
+    {
+      "epoch": 0.341,
+      "grad_norm": 17.0,
+      "learning_rate": 9.817695182423935e-06,
+      "loss": 2.6797,
+      "step": 3410
+    },
+    {
+      "epoch": 0.342,
+      "grad_norm": 21.0,
+      "learning_rate": 9.81624806415136e-06,
+      "loss": 2.7353,
+      "step": 3420
+    },
+    {
+      "epoch": 0.343,
+      "grad_norm": 16.125,
+      "learning_rate": 9.814795332522962e-06,
+      "loss": 2.5796,
+      "step": 3430
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 21.75,
+      "learning_rate": 9.813336989231901e-06,
+      "loss": 2.7384,
+      "step": 3440
+    },
+    {
+      "epoch": 0.345,
+      "grad_norm": 19.875,
+      "learning_rate": 9.811873035977883e-06,
+      "loss": 2.6481,
+      "step": 3450
+    },
+    {
+      "epoch": 0.346,
+      "grad_norm": 19.0,
+      "learning_rate": 9.810403474467149e-06,
+      "loss": 2.6103,
+      "step": 3460
+    },
+    {
+      "epoch": 0.347,
+      "grad_norm": 19.75,
+      "learning_rate": 9.808928306412482e-06,
+      "loss": 2.5278,
+      "step": 3470
+    },
+    {
+      "epoch": 0.348,
+      "grad_norm": 18.0,
+      "learning_rate": 9.807447533533196e-06,
+      "loss": 2.8516,
+      "step": 3480
+    },
+    {
+      "epoch": 0.349,
+      "grad_norm": 24.75,
+      "learning_rate": 9.805961157555134e-06,
+      "loss": 2.6572,
+      "step": 3490
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 16.25,
+      "learning_rate": 9.804469180210677e-06,
+      "loss": 2.4459,
+      "step": 3500
+    },
+    {
+      "epoch": 0.351,
+      "grad_norm": 18.875,
+      "learning_rate": 9.802971603238726e-06,
+      "loss": 2.6319,
+      "step": 3510
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 19.625,
+      "learning_rate": 9.801468428384716e-06,
+      "loss": 2.6243,
+      "step": 3520
+    },
+    {
+      "epoch": 0.353,
+      "grad_norm": 19.875,
+      "learning_rate": 9.799959657400604e-06,
+      "loss": 2.4742,
+      "step": 3530
+    },
+    {
+      "epoch": 0.354,
+      "grad_norm": 19.0,
+      "learning_rate": 9.798445292044865e-06,
+      "loss": 2.8687,
+      "step": 3540
+    },
+    {
+      "epoch": 0.355,
+      "grad_norm": 23.0,
+      "learning_rate": 9.796925334082501e-06,
+      "loss": 2.8029,
+      "step": 3550
+    },
+    {
+      "epoch": 0.356,
+      "grad_norm": 21.5,
+      "learning_rate": 9.795399785285026e-06,
+      "loss": 2.9439,
+      "step": 3560
+    },
+    {
+      "epoch": 0.357,
+      "grad_norm": 22.375,
+      "learning_rate": 9.793868647430476e-06,
+      "loss": 2.5769,
+      "step": 3570
+    },
+    {
+      "epoch": 0.358,
+      "grad_norm": 21.0,
+      "learning_rate": 9.792331922303396e-06,
+      "loss": 2.5519,
+      "step": 3580
+    },
+    {
+      "epoch": 0.359,
+      "grad_norm": 17.375,
+      "learning_rate": 9.790789611694846e-06,
+      "loss": 2.5939,
+      "step": 3590
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 21.375,
+      "learning_rate": 9.789241717402396e-06,
+      "loss": 2.3467,
+      "step": 3600
+    },
+    {
+      "epoch": 0.361,
+      "grad_norm": 20.5,
+      "learning_rate": 9.78768824123012e-06,
+      "loss": 2.506,
+      "step": 3610
+    },
+    {
+      "epoch": 0.362,
+      "grad_norm": 16.625,
+      "learning_rate": 9.786129184988601e-06,
+      "loss": 2.4018,
+      "step": 3620
+    },
+    {
+      "epoch": 0.363,
+      "grad_norm": 20.0,
+      "learning_rate": 9.78456455049493e-06,
+      "loss": 2.5399,
+      "step": 3630
+    },
+    {
+      "epoch": 0.364,
+      "grad_norm": 18.75,
+      "learning_rate": 9.782994339572688e-06,
+      "loss": 2.4026,
+      "step": 3640
+    },
+    {
+      "epoch": 0.365,
+      "grad_norm": 20.25,
+      "learning_rate": 9.781418554051964e-06,
+      "loss": 2.3732,
+      "step": 3650
+    },
+    {
+      "epoch": 0.366,
+      "grad_norm": 17.25,
+      "learning_rate": 9.779837195769345e-06,
+      "loss": 2.6251,
+      "step": 3660
+    },
+    {
+      "epoch": 0.367,
+      "grad_norm": 20.75,
+      "learning_rate": 9.778250266567908e-06,
+      "loss": 2.7751,
+      "step": 3670
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 23.0,
+      "learning_rate": 9.776657768297225e-06,
+      "loss": 2.6921,
+      "step": 3680
+    },
+    {
+      "epoch": 0.369,
+      "grad_norm": 17.5,
+      "learning_rate": 9.775059702813357e-06,
+      "loss": 2.4276,
+      "step": 3690
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 17.875,
+      "learning_rate": 9.77345607197886e-06,
+      "loss": 2.4817,
+      "step": 3700
+    },
+    {
+      "epoch": 0.371,
+      "grad_norm": 18.25,
+      "learning_rate": 9.771846877662768e-06,
+      "loss": 2.6489,
+      "step": 3710
+    },
+    {
+      "epoch": 0.372,
+      "grad_norm": 20.875,
+      "learning_rate": 9.770232121740606e-06,
+      "loss": 2.6174,
+      "step": 3720
+    },
+    {
+      "epoch": 0.373,
+      "grad_norm": 19.375,
+      "learning_rate": 9.768611806094376e-06,
+      "loss": 2.4813,
+      "step": 3730
+    },
+    {
+      "epoch": 0.374,
+      "grad_norm": 21.0,
+      "learning_rate": 9.766985932612566e-06,
+      "loss": 2.5242,
+      "step": 3740
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 21.125,
+      "learning_rate": 9.765354503190133e-06,
+      "loss": 2.5407,
+      "step": 3750
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 15.875,
+      "learning_rate": 9.763717519728518e-06,
+      "loss": 2.5035,
+      "step": 3760
+    },
+    {
+      "epoch": 0.377,
+      "grad_norm": 19.125,
+      "learning_rate": 9.76207498413563e-06,
+      "loss": 2.9284,
+      "step": 3770
+    },
+    {
+      "epoch": 0.378,
+      "grad_norm": 19.25,
+      "learning_rate": 9.760426898325849e-06,
+      "loss": 2.9116,
+      "step": 3780
+    },
+    {
+      "epoch": 0.379,
+      "grad_norm": 18.125,
+      "learning_rate": 9.758773264220027e-06,
+      "loss": 2.5067,
+      "step": 3790
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 20.25,
+      "learning_rate": 9.757114083745482e-06,
+      "loss": 2.3966,
+      "step": 3800
+    },
+    {
+      "epoch": 0.381,
+      "grad_norm": 18.125,
+      "learning_rate": 9.755449358835995e-06,
+      "loss": 2.7571,
+      "step": 3810
+    },
+    {
+      "epoch": 0.382,
+      "grad_norm": 18.0,
+      "learning_rate": 9.75377909143181e-06,
+      "loss": 2.5862,
+      "step": 3820
+    },
+    {
+      "epoch": 0.383,
+      "grad_norm": 19.5,
+      "learning_rate": 9.752103283479626e-06,
+      "loss": 2.46,
+      "step": 3830
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 19.25,
+      "learning_rate": 9.750421936932608e-06,
+      "loss": 2.3346,
+      "step": 3840
+    },
+    {
+      "epoch": 0.385,
+      "grad_norm": 19.5,
+      "learning_rate": 9.74873505375037e-06,
+      "loss": 2.4179,
+      "step": 3850
+    },
+    {
+      "epoch": 0.386,
+      "grad_norm": 17.75,
+      "learning_rate": 9.747042635898982e-06,
+      "loss": 2.3839,
+      "step": 3860
+    },
+    {
+      "epoch": 0.387,
+      "grad_norm": 14.75,
+      "learning_rate": 9.745344685350963e-06,
+      "loss": 2.646,
+      "step": 3870
+    },
+    {
+      "epoch": 0.388,
+      "grad_norm": 20.0,
+      "learning_rate": 9.743641204085282e-06,
+      "loss": 2.6814,
+      "step": 3880
+    },
+    {
+      "epoch": 0.389,
+      "grad_norm": 18.0,
+      "learning_rate": 9.74193219408735e-06,
+      "loss": 2.4887,
+      "step": 3890
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 18.0,
+      "learning_rate": 9.740217657349028e-06,
+      "loss": 2.6592,
+      "step": 3900
+    },
+    {
+      "epoch": 0.391,
+      "grad_norm": 20.25,
+      "learning_rate": 9.738497595868615e-06,
+      "loss": 2.4634,
+      "step": 3910
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 19.75,
+      "learning_rate": 9.736772011650846e-06,
+      "loss": 2.9073,
+      "step": 3920
+    },
+    {
+      "epoch": 0.393,
+      "grad_norm": 17.25,
+      "learning_rate": 9.735040906706898e-06,
+      "loss": 2.3341,
+      "step": 3930
+    },
+    {
+      "epoch": 0.394,
+      "grad_norm": 22.0,
+      "learning_rate": 9.733304283054384e-06,
+      "loss": 2.6865,
+      "step": 3940
+    },
+    {
+      "epoch": 0.395,
+      "grad_norm": 20.25,
+      "learning_rate": 9.731562142717341e-06,
+      "loss": 2.6186,
+      "step": 3950
+    },
+    {
+      "epoch": 0.396,
+      "grad_norm": 14.75,
+      "learning_rate": 9.729814487726241e-06,
+      "loss": 2.3226,
+      "step": 3960
+    },
+    {
+      "epoch": 0.397,
+      "grad_norm": 23.25,
+      "learning_rate": 9.728061320117985e-06,
+      "loss": 2.4437,
+      "step": 3970
+    },
+    {
+      "epoch": 0.398,
+      "grad_norm": 17.5,
+      "learning_rate": 9.726302641935895e-06,
+      "loss": 2.3598,
+      "step": 3980
+    },
+    {
+      "epoch": 0.399,
+      "grad_norm": 14.125,
+      "learning_rate": 9.724538455229718e-06,
+      "loss": 2.3781,
+      "step": 3990
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 21.5,
+      "learning_rate": 9.722768762055619e-06,
+      "loss": 2.8198,
+      "step": 4000
+    },
+    {
+      "epoch": 0.401,
+      "grad_norm": 18.25,
+      "learning_rate": 9.720993564476183e-06,
+      "loss": 2.2972,
+      "step": 4010
+    },
+    {
+      "epoch": 0.402,
+      "grad_norm": 20.25,
+      "learning_rate": 9.719212864560412e-06,
+      "loss": 2.399,
+      "step": 4020
+    },
+    {
+      "epoch": 0.403,
+      "grad_norm": 20.25,
+      "learning_rate": 9.717426664383713e-06,
+      "loss": 2.5677,
+      "step": 4030
+    },
+    {
+      "epoch": 0.404,
+      "grad_norm": 17.75,
+      "learning_rate": 9.715634966027915e-06,
+      "loss": 2.6581,
+      "step": 4040
+    },
+    {
+      "epoch": 0.405,
+      "grad_norm": 20.5,
+      "learning_rate": 9.713837771581248e-06,
+      "loss": 2.5335,
+      "step": 4050
+    },
+    {
+      "epoch": 0.406,
+      "grad_norm": 17.75,
+      "learning_rate": 9.712035083138347e-06,
+      "loss": 2.1949,
+      "step": 4060
+    },
+    {
+      "epoch": 0.407,
+      "grad_norm": 21.0,
+      "learning_rate": 9.710226902800255e-06,
+      "loss": 2.3183,
+      "step": 4070
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 20.5,
+      "learning_rate": 9.708413232674412e-06,
+      "loss": 2.3879,
+      "step": 4080
+    },
+    {
+      "epoch": 0.409,
+      "grad_norm": 17.125,
+      "learning_rate": 9.706594074874656e-06,
+      "loss": 2.3861,
+      "step": 4090
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 20.375,
+      "learning_rate": 9.704769431521226e-06,
+      "loss": 2.4686,
+      "step": 4100
+    },
+    {
+      "epoch": 0.411,
+      "grad_norm": 14.9375,
+      "learning_rate": 9.702939304740747e-06,
+      "loss": 2.4789,
+      "step": 4110
+    },
+    {
+      "epoch": 0.412,
+      "grad_norm": 20.0,
+      "learning_rate": 9.701103696666241e-06,
+      "loss": 2.5005,
+      "step": 4120
+    },
+    {
+      "epoch": 0.413,
+      "grad_norm": 16.75,
+      "learning_rate": 9.699262609437117e-06,
+      "loss": 2.6703,
+      "step": 4130
+    },
+    {
+      "epoch": 0.414,
+      "grad_norm": 17.875,
+      "learning_rate": 9.697416045199167e-06,
+      "loss": 2.6555,
+      "step": 4140
+    },
+    {
+      "epoch": 0.415,
+      "grad_norm": 19.625,
+      "learning_rate": 9.695564006104572e-06,
+      "loss": 2.5342,
+      "step": 4150
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 20.625,
+      "learning_rate": 9.693706494311887e-06,
+      "loss": 2.3838,
+      "step": 4160
+    },
+    {
+      "epoch": 0.417,
+      "grad_norm": 15.75,
+      "learning_rate": 9.691843511986051e-06,
+      "loss": 2.4372,
+      "step": 4170
+    },
+    {
+      "epoch": 0.418,
+      "grad_norm": 20.5,
+      "learning_rate": 9.68997506129838e-06,
+      "loss": 2.6174,
+      "step": 4180
+    },
+    {
+      "epoch": 0.419,
+      "grad_norm": 16.625,
+      "learning_rate": 9.688101144426557e-06,
+      "loss": 2.3661,
+      "step": 4190
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 18.5,
+      "learning_rate": 9.68622176355464e-06,
+      "loss": 2.1973,
+      "step": 4200
+    },
+    {
+      "epoch": 0.421,
+      "grad_norm": 16.25,
+      "learning_rate": 9.684336920873055e-06,
+      "loss": 2.3309,
+      "step": 4210
+    },
+    {
+      "epoch": 0.422,
+      "grad_norm": 17.0,
+      "learning_rate": 9.682446618578593e-06,
+      "loss": 2.4981,
+      "step": 4220
+    },
+    {
+      "epoch": 0.423,
+      "grad_norm": 14.625,
+      "learning_rate": 9.680550858874409e-06,
+      "loss": 2.8275,
+      "step": 4230
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 20.5,
+      "learning_rate": 9.678649643970018e-06,
+      "loss": 2.6569,
+      "step": 4240
+    },
+    {
+      "epoch": 0.425,
+      "grad_norm": 19.875,
+      "learning_rate": 9.676742976081295e-06,
+      "loss": 2.8605,
+      "step": 4250
+    },
+    {
+      "epoch": 0.426,
+      "grad_norm": 17.375,
+      "learning_rate": 9.674830857430466e-06,
+      "loss": 2.396,
+      "step": 4260
+    },
+    {
+      "epoch": 0.427,
+      "grad_norm": 17.25,
+      "learning_rate": 9.672913290246115e-06,
+      "loss": 2.4021,
+      "step": 4270
+    },
+    {
+      "epoch": 0.428,
+      "grad_norm": 20.625,
+      "learning_rate": 9.670990276763174e-06,
+      "loss": 2.5815,
+      "step": 4280
+    },
+    {
+      "epoch": 0.429,
+      "grad_norm": 17.75,
+      "learning_rate": 9.669061819222923e-06,
+      "loss": 2.3317,
+      "step": 4290
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 22.625,
+      "learning_rate": 9.667127919872983e-06,
+      "loss": 2.6058,
+      "step": 4300
+    },
+    {
+      "epoch": 0.431,
+      "grad_norm": 20.625,
+      "learning_rate": 9.665188580967326e-06,
+      "loss": 2.7117,
+      "step": 4310
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 19.625,
+      "learning_rate": 9.663243804766256e-06,
+      "loss": 2.5155,
+      "step": 4320
+    },
+    {
+      "epoch": 0.433,
+      "grad_norm": 19.375,
+      "learning_rate": 9.661293593536418e-06,
+      "loss": 2.5754,
+      "step": 4330
+    },
+    {
+      "epoch": 0.434,
+      "grad_norm": 19.25,
+      "learning_rate": 9.65933794955079e-06,
+      "loss": 2.6044,
+      "step": 4340
+    },
+    {
+      "epoch": 0.435,
+      "grad_norm": 18.75,
+      "learning_rate": 9.657376875088687e-06,
+      "loss": 2.4606,
+      "step": 4350
+    },
+    {
+      "epoch": 0.436,
+      "grad_norm": 18.875,
+      "learning_rate": 9.655410372435743e-06,
+      "loss": 2.5395,
+      "step": 4360
+    },
+    {
+      "epoch": 0.437,
+      "grad_norm": 19.0,
+      "learning_rate": 9.653438443883928e-06,
+      "loss": 2.4258,
+      "step": 4370
+    },
+    {
+      "epoch": 0.438,
+      "grad_norm": 19.125,
+      "learning_rate": 9.651461091731531e-06,
+      "loss": 2.3891,
+      "step": 4380
+    },
+    {
+      "epoch": 0.439,
+      "grad_norm": 20.125,
+      "learning_rate": 9.649478318283163e-06,
+      "loss": 2.7879,
+      "step": 4390
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 19.125,
+      "learning_rate": 9.647490125849756e-06,
+      "loss": 2.3331,
+      "step": 4400
+    },
+    {
+      "epoch": 0.441,
+      "grad_norm": 21.875,
+      "learning_rate": 9.645496516748555e-06,
+      "loss": 2.4382,
+      "step": 4410
+    },
+    {
+      "epoch": 0.442,
+      "grad_norm": 20.75,
+      "learning_rate": 9.64349749330312e-06,
+      "loss": 2.5799,
+      "step": 4420
+    },
+    {
+      "epoch": 0.443,
+      "grad_norm": 17.25,
+      "learning_rate": 9.64149305784332e-06,
+      "loss": 2.7297,
+      "step": 4430
+    },
+    {
+      "epoch": 0.444,
+      "grad_norm": 17.875,
+      "learning_rate": 9.639483212705331e-06,
+      "loss": 2.7431,
+      "step": 4440
+    },
+    {
+      "epoch": 0.445,
+      "grad_norm": 19.0,
+      "learning_rate": 9.637467960231637e-06,
+      "loss": 2.4937,
+      "step": 4450
+    },
+    {
+      "epoch": 0.446,
+      "grad_norm": 25.375,
+      "learning_rate": 9.635447302771022e-06,
+      "loss": 2.6898,
+      "step": 4460
+    },
+    {
+      "epoch": 0.447,
+      "grad_norm": 18.875,
+      "learning_rate": 9.63342124267857e-06,
+      "loss": 2.2197,
+      "step": 4470
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 20.125,
+      "learning_rate": 9.631389782315663e-06,
+      "loss": 2.5553,
+      "step": 4480
+    },
+    {
+      "epoch": 0.449,
+      "grad_norm": 18.0,
+      "learning_rate": 9.629352924049975e-06,
+      "loss": 2.477,
+      "step": 4490
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 16.75,
+      "learning_rate": 9.627310670255471e-06,
+      "loss": 2.2515,
+      "step": 4500
+    },
+    {
+      "epoch": 0.451,
+      "grad_norm": 19.625,
+      "learning_rate": 9.625263023312408e-06,
+      "loss": 2.5055,
+      "step": 4510
+    },
+    {
+      "epoch": 0.452,
+      "grad_norm": 15.75,
+      "learning_rate": 9.623209985607327e-06,
+      "loss": 2.3196,
+      "step": 4520
+    },
+    {
+      "epoch": 0.453,
+      "grad_norm": 19.125,
+      "learning_rate": 9.621151559533048e-06,
+      "loss": 2.4119,
+      "step": 4530
+    },
+    {
+      "epoch": 0.454,
+      "grad_norm": 20.625,
+      "learning_rate": 9.619087747488675e-06,
+      "loss": 2.7065,
+      "step": 4540
+    },
+    {
+      "epoch": 0.455,
+      "grad_norm": 19.375,
+      "learning_rate": 9.617018551879593e-06,
+      "loss": 2.531,
+      "step": 4550
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 19.625,
+      "learning_rate": 9.614943975117454e-06,
+      "loss": 2.7495,
+      "step": 4560
+    },
+    {
+      "epoch": 0.457,
+      "grad_norm": 16.25,
+      "learning_rate": 9.612864019620187e-06,
+      "loss": 2.454,
+      "step": 4570
+    },
+    {
+      "epoch": 0.458,
+      "grad_norm": 19.75,
+      "learning_rate": 9.610778687811988e-06,
+      "loss": 2.5686,
+      "step": 4580
+    },
+    {
+      "epoch": 0.459,
+      "grad_norm": 22.125,
+      "learning_rate": 9.608687982123318e-06,
+      "loss": 2.4886,
+      "step": 4590
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 15.4375,
+      "learning_rate": 9.606591904990903e-06,
+      "loss": 2.3279,
+      "step": 4600
+    },
+    {
+      "epoch": 0.461,
+      "grad_norm": 15.625,
+      "learning_rate": 9.60449045885773e-06,
+      "loss": 2.9503,
+      "step": 4610
+    },
+    {
+      "epoch": 0.462,
+      "grad_norm": 21.0,
+      "learning_rate": 9.602383646173041e-06,
+      "loss": 2.5394,
+      "step": 4620
+    },
+    {
+      "epoch": 0.463,
+      "grad_norm": 19.75,
+      "learning_rate": 9.600271469392335e-06,
+      "loss": 2.4151,
+      "step": 4630
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 20.875,
+      "learning_rate": 9.598153930977364e-06,
+      "loss": 2.7415,
+      "step": 4640
+    },
+    {
+      "epoch": 0.465,
+      "grad_norm": 18.75,
+      "learning_rate": 9.596031033396126e-06,
+      "loss": 2.5966,
+      "step": 4650
+    },
+    {
+      "epoch": 0.466,
+      "grad_norm": 18.625,
+      "learning_rate": 9.593902779122865e-06,
+      "loss": 2.4929,
+      "step": 4660
+    },
+    {
+      "epoch": 0.467,
+      "grad_norm": 21.0,
+      "learning_rate": 9.591769170638072e-06,
+      "loss": 2.6185,
+      "step": 4670
+    },
+    {
+      "epoch": 0.468,
+      "grad_norm": 19.25,
+      "learning_rate": 9.589630210428476e-06,
+      "loss": 2.5501,
+      "step": 4680
+    },
+    {
+      "epoch": 0.469,
+      "grad_norm": 18.125,
+      "learning_rate": 9.587485900987042e-06,
+      "loss": 2.4147,
+      "step": 4690
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 17.5,
+      "learning_rate": 9.585336244812969e-06,
+      "loss": 2.554,
+      "step": 4700
+    },
+    {
+      "epoch": 0.471,
+      "grad_norm": 17.125,
+      "learning_rate": 9.583181244411694e-06,
+      "loss": 2.64,
+      "step": 4710
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 17.5,
+      "learning_rate": 9.581020902294876e-06,
+      "loss": 2.5522,
+      "step": 4720
+    },
+    {
+      "epoch": 0.473,
+      "grad_norm": 24.625,
+      "learning_rate": 9.5788552209804e-06,
+      "loss": 2.8493,
+      "step": 4730
+    },
+    {
+      "epoch": 0.474,
+      "grad_norm": 19.5,
+      "learning_rate": 9.576684202992381e-06,
+      "loss": 2.3638,
+      "step": 4740
+    },
+    {
+      "epoch": 0.475,
+      "grad_norm": 18.75,
+      "learning_rate": 9.574507850861145e-06,
+      "loss": 2.4673,
+      "step": 4750
+    },
+    {
+      "epoch": 0.476,
+      "grad_norm": 20.0,
+      "learning_rate": 9.572326167123242e-06,
+      "loss": 2.5699,
+      "step": 4760
+    },
+    {
+      "epoch": 0.477,
+      "grad_norm": 18.0,
+      "learning_rate": 9.570139154321429e-06,
+      "loss": 2.522,
+      "step": 4770
+    },
+    {
+      "epoch": 0.478,
+      "grad_norm": 19.5,
+      "learning_rate": 9.567946815004681e-06,
+      "loss": 2.7148,
+      "step": 4780
+    },
+    {
+      "epoch": 0.479,
+      "grad_norm": 17.75,
+      "learning_rate": 9.565749151728176e-06,
+      "loss": 2.5433,
+      "step": 4790
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 19.375,
+      "learning_rate": 9.563546167053302e-06,
+      "loss": 2.6228,
+      "step": 4800
+    },
+    {
+      "epoch": 0.481,
+      "grad_norm": 19.75,
+      "learning_rate": 9.561337863547642e-06,
+      "loss": 2.3278,
+      "step": 4810
+    },
+    {
+      "epoch": 0.482,
+      "grad_norm": 18.125,
+      "learning_rate": 9.559124243784985e-06,
+      "loss": 2.318,
+      "step": 4820
+    },
+    {
+      "epoch": 0.483,
+      "grad_norm": 17.5,
+      "learning_rate": 9.556905310345313e-06,
+      "loss": 2.6863,
+      "step": 4830
+    },
+    {
+      "epoch": 0.484,
+      "grad_norm": 16.625,
+      "learning_rate": 9.554681065814802e-06,
+      "loss": 2.6111,
+      "step": 4840
+    },
+    {
+      "epoch": 0.485,
+      "grad_norm": 16.375,
+      "learning_rate": 9.552451512785816e-06,
+      "loss": 2.3087,
+      "step": 4850
+    },
+    {
+      "epoch": 0.486,
+      "grad_norm": 18.625,
+      "learning_rate": 9.55021665385691e-06,
+      "loss": 2.5179,
+      "step": 4860
+    },
+    {
+      "epoch": 0.487,
+      "grad_norm": 19.75,
+      "learning_rate": 9.54797649163282e-06,
+      "loss": 2.4464,
+      "step": 4870
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 18.25,
+      "learning_rate": 9.545731028724463e-06,
+      "loss": 2.6145,
+      "step": 4880
+    },
+    {
+      "epoch": 0.489,
+      "grad_norm": 21.0,
+      "learning_rate": 9.543480267748935e-06,
+      "loss": 2.4499,
+      "step": 4890
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 20.625,
+      "learning_rate": 9.541224211329504e-06,
+      "loss": 2.3893,
+      "step": 4900
+    },
+    {
+      "epoch": 0.491,
+      "grad_norm": 17.75,
+      "learning_rate": 9.53896286209562e-06,
+      "loss": 2.475,
+      "step": 4910
+    },
+    {
+      "epoch": 0.492,
+      "grad_norm": 16.375,
+      "learning_rate": 9.536696222682887e-06,
+      "loss": 2.585,
+      "step": 4920
+    },
+    {
+      "epoch": 0.493,
+      "grad_norm": 21.25,
+      "learning_rate": 9.534424295733085e-06,
+      "loss": 2.4422,
+      "step": 4930
+    },
+    {
+      "epoch": 0.494,
+      "grad_norm": 21.75,
+      "learning_rate": 9.532147083894153e-06,
+      "loss": 2.7116,
+      "step": 4940
+    },
+    {
+      "epoch": 0.495,
+      "grad_norm": 20.5,
+      "learning_rate": 9.529864589820192e-06,
+      "loss": 2.7823,
+      "step": 4950
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 16.0,
+      "learning_rate": 9.527576816171454e-06,
+      "loss": 2.7586,
+      "step": 4960
+    },
+    {
+      "epoch": 0.497,
+      "grad_norm": 20.25,
+      "learning_rate": 9.525283765614353e-06,
+      "loss": 2.4244,
+      "step": 4970
+    },
+    {
+      "epoch": 0.498,
+      "grad_norm": 19.0,
+      "learning_rate": 9.522985440821444e-06,
+      "loss": 2.7349,
+      "step": 4980
+    },
+    {
+      "epoch": 0.499,
+      "grad_norm": 22.625,
+      "learning_rate": 9.520681844471435e-06,
+      "loss": 2.7601,
+      "step": 4990
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 18.625,
+      "learning_rate": 9.518372979249178e-06,
+      "loss": 2.7425,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}