diff --git a/.gitattributes b/.gitattributes
index 1518925937092a5a4822c82654b2fc2073fa7833..519c4fbd74228d644a85339b65fe7196d630f5f4 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -738,3 +738,12 @@ checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-500/tokeni
 checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2409/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/2d73a6863086ff9d491c28e49df9fb697cd92c2b.lock b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/2d73a6863086ff9d491c28e49df9fb697cd92c2b.lock
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb.lock b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb.lock
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/47d4a5aa69cdef91a53b77f5c5583647a578ca0e.lock b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/47d4a5aa69cdef91a53b77f5c5583647a578ca0e.lock
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/584d8d3e3f82f7964955174dfe5e3b1cf117a9d859f022cfdf7fcb884856e002.lock b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/584d8d3e3f82f7964955174dfe5e3b1cf117a9d859f022cfdf7fcb884856e002.lock
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58.lock b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/.locks/models--meta-llama--Llama-3.2-3B/d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58.lock
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/.no_exist/13afe5124825b4f3751f836b40dafda64c1ed062/adapter_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/.no_exist/13afe5124825b4f3751f836b40dafda64c1ed062/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/.no_exist/13afe5124825b4f3751f836b40dafda64c1ed062/model.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/.no_exist/13afe5124825b4f3751f836b40dafda64c1ed062/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/2d73a6863086ff9d491c28e49df9fb697cd92c2b b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/2d73a6863086ff9d491c28e49df9fb697cd92c2b
new file mode 100644
index 0000000000000000000000000000000000000000..2d73a6863086ff9d491c28e49df9fb697cd92c2b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/2d73a6863086ff9d491c28e49df9fb697cd92c2b
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.45.0.dev0",
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_p": 0.9
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb
new file mode 100644
index 0000000000000000000000000000000000000000..82a877359be75c9c7ad1226b362efeeabf2edef4
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb
+size 1459729952
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/47d4a5aa69cdef91a53b77f5c5583647a578ca0e b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/47d4a5aa69cdef91a53b77f5c5583647a578ca0e
new file mode 100644
index 0000000000000000000000000000000000000000..47d4a5aa69cdef91a53b77f5c5583647a578ca0e
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/47d4a5aa69cdef91a53b77f5c5583647a578ca0e
@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58 b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58
new file mode 100644
index 0000000000000000000000000000000000000000..d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/blobs/d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58
@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 6425499648
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/refs/main b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/refs/main
new file mode 100644
index 0000000000000000000000000000000000000000..940c46e96db631910d2cc5186ab76cd83e32b124
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/refs/main
@@ -0,0 +1 @@
+13afe5124825b4f3751f836b40dafda64c1ed062
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..47d4a5aa69cdef91a53b77f5c5583647a578ca0e
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/config.json
@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d73a6863086ff9d491c28e49df9fb697cd92c2b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.45.0.dev0",
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_p": 0.9
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..82a877359be75c9c7ad1226b362efeeabf2edef4
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4719a04514ec2f060240711b7c33ab21187cac730ecaba3040b7a0fd95a9cefb
+size 1459729952
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3a1f0f5f401eeadca0c7a6786bd9e877fd42e58
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/artifacts/models--meta-llama--Llama-3.2-3B/snapshots/13afe5124825b4f3751f836b40dafda64c1ed062/model.safetensors.index.json
@@ -0,0 +1,261 @@
+{
+  "metadata": {
+    "total_size": 6425499648
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1000/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1000/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1000/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fb961e8562cc582e56eef9a7a0047ec14022b3c2
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1100/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0b6906b1b7bd0c56a4dcb57bd84818742ebfdfd3e5758f89d884c29b52c94e
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..40c21cd2e02cf6ee6ee7aeb48c8791fcf90d4aa8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1300/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10985ee85a9c8562f43d68e3a62a7e7072bc2a91107bd154308c504beb57af06
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c6549714ec0b99d6f60b380a18005a5fe86ca36f
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1400/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4992428517d685ab13f0e3de31be427326ac6979f32f900c84d023a8bea85538
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..21efc2152751649f0f05a5e0ad983dc600caeabd
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1500/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc57ebab56f6f3996f3ee6cf11308bcfcfa92e2f3e5ac4c7aa1cc16f68c5e036
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6369909c5205b7328c2f7cd58d88bd16a1049e61
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1600/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed06770ffc52e4cb42197bdc48589d750e80f6719b9b3257d921020931ee366a
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1900/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1900/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..26ee7760836c71714960369955a1c6bb42f9a29a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-1900/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:897c1083d853b3d66efcaa043a6f7bfad34fbb43cad845595029823b5530a000
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9829253a45a30486360d23ef44a31e49a0fbb37c
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1200c6256bb3c5211e1fa1cfcea6721e1b8871fd3c8953a59bf49a59f67404c
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ed05b8eb12ac744b29eb407adca7ab9af789a745
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b59dfa224b09069bb1265a6a5cb86c22c34f53669b17ebd9335621f959bf7a4
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a26252516294710c0550a7751d8c4bc072b73762
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:316a81119e1ab6e1b3e5b9d890ce1d0ca9ea9223094844fdcf8b7d9f0f4bb6c2
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09627fb7af41a998516298027fb2b99a824026c0
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a49c33a964ee4724c018b0c4d2cc677c6cccdb89a812ee1343b9cc0666541f0
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6
+size 17210734
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..fddeacce09fc499aa875828237d2a762f0a87f04
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/trainer_state.json
@@ -0,0 +1,16413 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.613565650280025,
+  "eval_steps": 10,
+  "global_step": 2100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8368,
+      "step": 1
+    },
+    {
+      "epoch": 0.002489110143123833,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.86,
+      "step": 2
+    },
+    {
+      "epoch": 0.00373366521468575,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8265,
+      "step": 3
+    },
+    {
+      "epoch": 0.004978220286247666,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8271,
+      "step": 4
+    },
+    {
+      "epoch": 0.006222775357809583,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8768,
+      "step": 5
+    },
+    {
+      "epoch": 0.0074673304293715,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8402,
+      "step": 6
+    },
+    {
+      "epoch": 0.008711885500933417,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.785,
+      "step": 7
+    },
+    {
+      "epoch": 0.009956440572495333,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8466,
+      "step": 8
+    },
+    {
+      "epoch": 0.01120099564405725,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.7673,
+      "step": 9
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "eval_loss": 2.8535053730010986,
+      "eval_runtime": 42.446,
+      "eval_samples_per_second": 23.559,
+      "eval_steps_per_second": 0.989,
+      "step": 10
+    },
+    {
+      "epoch": 0.013690105787181083,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8592,
+      "step": 11
+    },
+    {
+      "epoch": 0.014934660858743,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.804,
+      "step": 12
+    },
+    {
+      "epoch": 0.016179215930304917,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8076,
+      "step": 13
+    },
+    {
+      "epoch": 0.017423771001866834,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8219,
+      "step": 14
+    },
+    {
+      "epoch": 0.018668326073428748,
+      "grad_norm": 5.105710506439209,
+      "learning_rate": 2.0746887966804982e-08,
+      "loss": 2.7995,
+      "step": 15
+    },
+    {
+      "epoch": 0.019912881144990666,
+      "grad_norm": 5.156588077545166,
+      "learning_rate": 4.1493775933609963e-08,
+      "loss": 2.8577,
+      "step": 16
+    },
+    {
+      "epoch": 0.021157436216552583,
+      "grad_norm": 5.709373950958252,
+      "learning_rate": 6.224066390041494e-08,
+      "loss": 2.8433,
+      "step": 17
+    },
+    {
+      "epoch": 0.0224019912881145,
+      "grad_norm": 5.64658784866333,
+      "learning_rate": 8.298755186721993e-08,
+      "loss": 2.8428,
+      "step": 18
+    },
+    {
+      "epoch": 0.023646546359676415,
+      "grad_norm": 5.541937351226807,
+      "learning_rate": 1.037344398340249e-07,
+      "loss": 2.7785,
+      "step": 19
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "grad_norm": 5.233374118804932,
+      "learning_rate": 1.2448132780082988e-07,
+      "loss": 2.8599,
+      "step": 20
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "eval_loss": 2.8521182537078857,
+      "eval_runtime": 43.5664,
+      "eval_samples_per_second": 22.953,
+      "eval_steps_per_second": 0.964,
+      "step": 20
+    },
+    {
+      "epoch": 0.02613565650280025,
+      "grad_norm": 5.407598972320557,
+      "learning_rate": 1.4522821576763488e-07,
+      "loss": 2.8081,
+      "step": 21
+    },
+    {
+      "epoch": 0.027380211574362167,
+      "grad_norm": 5.0483317375183105,
+      "learning_rate": 1.6597510373443985e-07,
+      "loss": 2.8489,
+      "step": 22
+    },
+    {
+      "epoch": 0.02862476664592408,
+      "grad_norm": 4.985588550567627,
+      "learning_rate": 1.8672199170124483e-07,
+      "loss": 2.8375,
+      "step": 23
+    },
+    {
+      "epoch": 0.029869321717486,
+      "grad_norm": 5.201700687408447,
+      "learning_rate": 2.074688796680498e-07,
+      "loss": 2.8355,
+      "step": 24
+    },
+    {
+      "epoch": 0.031113876789047916,
+      "grad_norm": 5.009235382080078,
+      "learning_rate": 2.2821576763485478e-07,
+      "loss": 2.7774,
+      "step": 25
+    },
+    {
+      "epoch": 0.03235843186060983,
+      "grad_norm": 4.509210109710693,
+      "learning_rate": 2.4896265560165975e-07,
+      "loss": 2.7992,
+      "step": 26
+    },
+    {
+      "epoch": 0.03360298693217175,
+      "grad_norm": 4.529262065887451,
+      "learning_rate": 2.6970954356846476e-07,
+      "loss": 2.836,
+      "step": 27
+    },
+    {
+      "epoch": 0.03484754200373367,
+      "grad_norm": 4.526787281036377,
+      "learning_rate": 2.9045643153526976e-07,
+      "loss": 2.8128,
+      "step": 28
+    },
+    {
+      "epoch": 0.03609209707529558,
+      "grad_norm": 4.016609191894531,
+      "learning_rate": 3.112033195020747e-07,
+      "loss": 2.7947,
+      "step": 29
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "grad_norm": 3.4784562587738037,
+      "learning_rate": 3.319502074688797e-07,
+      "loss": 2.7854,
+      "step": 30
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "eval_loss": 2.8020100593566895,
+      "eval_runtime": 44.8464,
+      "eval_samples_per_second": 22.298,
+      "eval_steps_per_second": 0.937,
+      "step": 30
+    },
+    {
+      "epoch": 0.038581207218419414,
+      "grad_norm": 3.4887945652008057,
+      "learning_rate": 3.5269709543568466e-07,
+      "loss": 2.7908,
+      "step": 31
+    },
+    {
+      "epoch": 0.03982576228998133,
+      "grad_norm": 3.4533374309539795,
+      "learning_rate": 3.7344398340248966e-07,
+      "loss": 2.7358,
+      "step": 32
+    },
+    {
+      "epoch": 0.04107031736154325,
+      "grad_norm": 3.270954132080078,
+      "learning_rate": 3.941908713692946e-07,
+      "loss": 2.8013,
+      "step": 33
+    },
+    {
+      "epoch": 0.042314872433105166,
+      "grad_norm": 3.61322283744812,
+      "learning_rate": 4.149377593360996e-07,
+      "loss": 2.7521,
+      "step": 34
+    },
+    {
+      "epoch": 0.043559427504667084,
+      "grad_norm": 3.158719062805176,
+      "learning_rate": 4.3568464730290456e-07,
+      "loss": 2.7573,
+      "step": 35
+    },
+    {
+      "epoch": 0.044803982576229,
+      "grad_norm": 2.888746500015259,
+      "learning_rate": 4.5643153526970956e-07,
+      "loss": 2.7795,
+      "step": 36
+    },
+    {
+      "epoch": 0.04604853764779091,
+      "grad_norm": 3.022629499435425,
+      "learning_rate": 4.771784232365145e-07,
+      "loss": 2.7891,
+      "step": 37
+    },
+    {
+      "epoch": 0.04729309271935283,
+      "grad_norm": 3.038097381591797,
+      "learning_rate": 4.979253112033195e-07,
+      "loss": 2.7376,
+      "step": 38
+    },
+    {
+      "epoch": 0.04853764779091475,
+      "grad_norm": 2.9992239475250244,
+      "learning_rate": 5.186721991701245e-07,
+      "loss": 2.7279,
+      "step": 39
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "grad_norm": 2.908842086791992,
+      "learning_rate": 5.394190871369295e-07,
+      "loss": 2.6582,
+      "step": 40
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "eval_loss": 2.725968837738037,
+      "eval_runtime": 44.9498,
+      "eval_samples_per_second": 22.247,
+      "eval_steps_per_second": 0.934,
+      "step": 40
+    },
+    {
+      "epoch": 0.05102675793403858,
+      "grad_norm": 2.8025174140930176,
+      "learning_rate": 5.601659751037345e-07,
+      "loss": 2.6809,
+      "step": 41
+    },
+    {
+      "epoch": 0.0522713130056005,
+      "grad_norm": 2.7555699348449707,
+      "learning_rate": 5.809128630705395e-07,
+      "loss": 2.6954,
+      "step": 42
+    },
+    {
+      "epoch": 0.053515868077162417,
+      "grad_norm": 2.4732933044433594,
+      "learning_rate": 6.016597510373444e-07,
+      "loss": 2.7307,
+      "step": 43
+    },
+    {
+      "epoch": 0.054760423148724334,
+      "grad_norm": 2.451366424560547,
+      "learning_rate": 6.224066390041494e-07,
+      "loss": 2.7064,
+      "step": 44
+    },
+    {
+      "epoch": 0.056004978220286245,
+      "grad_norm": 2.5768678188323975,
+      "learning_rate": 6.431535269709543e-07,
+      "loss": 2.6466,
+      "step": 45
+    },
+    {
+      "epoch": 0.05724953329184816,
+      "grad_norm": 2.579332113265991,
+      "learning_rate": 6.639004149377594e-07,
+      "loss": 2.6127,
+      "step": 46
+    },
+    {
+      "epoch": 0.05849408836341008,
+      "grad_norm": 2.231207847595215,
+      "learning_rate": 6.846473029045644e-07,
+      "loss": 2.6429,
+      "step": 47
+    },
+    {
+      "epoch": 0.059738643434972,
+      "grad_norm": 2.134375810623169,
+      "learning_rate": 7.053941908713693e-07,
+      "loss": 2.6423,
+      "step": 48
+    },
+    {
+      "epoch": 0.060983198506533914,
+      "grad_norm": 2.0485494136810303,
+      "learning_rate": 7.261410788381744e-07,
+      "loss": 2.581,
+      "step": 49
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "grad_norm": 2.02909517288208,
+      "learning_rate": 7.468879668049793e-07,
+      "loss": 2.6357,
+      "step": 50
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "eval_loss": 2.632822036743164,
+      "eval_runtime": 47.0104,
+      "eval_samples_per_second": 21.272,
+      "eval_steps_per_second": 0.893,
+      "step": 50
+    },
+    {
+      "epoch": 0.06347230864965775,
+      "grad_norm": 1.892500877380371,
+      "learning_rate": 7.676348547717843e-07,
+      "loss": 2.5945,
+      "step": 51
+    },
+    {
+      "epoch": 0.06471686372121967,
+      "grad_norm": 2.107921600341797,
+      "learning_rate": 7.883817427385892e-07,
+      "loss": 2.5867,
+      "step": 52
+    },
+    {
+      "epoch": 0.06596141879278158,
+      "grad_norm": 1.8287527561187744,
+      "learning_rate": 8.091286307053943e-07,
+      "loss": 2.5862,
+      "step": 53
+    },
+    {
+      "epoch": 0.0672059738643435,
+      "grad_norm": 1.7342065572738647,
+      "learning_rate": 8.298755186721992e-07,
+      "loss": 2.5735,
+      "step": 54
+    },
+    {
+      "epoch": 0.06845052893590542,
+      "grad_norm": 1.7905038595199585,
+      "learning_rate": 8.506224066390042e-07,
+      "loss": 2.572,
+      "step": 55
+    },
+    {
+      "epoch": 0.06969508400746734,
+      "grad_norm": 1.9715144634246826,
+      "learning_rate": 8.713692946058091e-07,
+      "loss": 2.5797,
+      "step": 56
+    },
+    {
+      "epoch": 0.07093963907902924,
+      "grad_norm": 1.788966417312622,
+      "learning_rate": 8.921161825726142e-07,
+      "loss": 2.5567,
+      "step": 57
+    },
+    {
+      "epoch": 0.07218419415059116,
+      "grad_norm": 1.8978915214538574,
+      "learning_rate": 9.128630705394191e-07,
+      "loss": 2.5776,
+      "step": 58
+    },
+    {
+      "epoch": 0.07342874922215308,
+      "grad_norm": 1.6401689052581787,
+      "learning_rate": 9.336099585062241e-07,
+      "loss": 2.5662,
+      "step": 59
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "grad_norm": 1.8200798034667969,
+      "learning_rate": 9.54356846473029e-07,
+      "loss": 2.5001,
+      "step": 60
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "eval_loss": 2.569110870361328,
+      "eval_runtime": 47.3292,
+      "eval_samples_per_second": 21.129,
+      "eval_steps_per_second": 0.887,
+      "step": 60
+    },
+    {
+      "epoch": 0.07591785936527691,
+      "grad_norm": 1.5468674898147583,
+      "learning_rate": 9.751037344398341e-07,
+      "loss": 2.5308,
+      "step": 61
+    },
+    {
+      "epoch": 0.07716241443683883,
+      "grad_norm": 1.723684310913086,
+      "learning_rate": 9.95850622406639e-07,
+      "loss": 2.4907,
+      "step": 62
+    },
+    {
+      "epoch": 0.07840696950840075,
+      "grad_norm": 1.6346293687820435,
+      "learning_rate": 1.0165975103734441e-06,
+      "loss": 2.514,
+      "step": 63
+    },
+    {
+      "epoch": 0.07965152457996266,
+      "grad_norm": 1.8902088403701782,
+      "learning_rate": 1.037344398340249e-06,
+      "loss": 2.4859,
+      "step": 64
+    },
+    {
+      "epoch": 0.08089607965152458,
+      "grad_norm": 2.092611312866211,
+      "learning_rate": 1.058091286307054e-06,
+      "loss": 2.4888,
+      "step": 65
+    },
+    {
+      "epoch": 0.0821406347230865,
+      "grad_norm": 1.758482575416565,
+      "learning_rate": 1.078838174273859e-06,
+      "loss": 2.4537,
+      "step": 66
+    },
+    {
+      "epoch": 0.08338518979464841,
+      "grad_norm": 1.8941714763641357,
+      "learning_rate": 1.099585062240664e-06,
+      "loss": 2.5416,
+      "step": 67
+    },
+    {
+      "epoch": 0.08462974486621033,
+      "grad_norm": 1.7030234336853027,
+      "learning_rate": 1.120331950207469e-06,
+      "loss": 2.4791,
+      "step": 68
+    },
+    {
+      "epoch": 0.08587429993777225,
+      "grad_norm": 1.7602269649505615,
+      "learning_rate": 1.141078838174274e-06,
+      "loss": 2.4316,
+      "step": 69
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "grad_norm": 1.7635431289672852,
+      "learning_rate": 1.161825726141079e-06,
+      "loss": 2.4674,
+      "step": 70
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "eval_loss": 2.4922080039978027,
+      "eval_runtime": 45.3028,
+      "eval_samples_per_second": 22.074,
+      "eval_steps_per_second": 0.927,
+      "step": 70
+    },
+    {
+      "epoch": 0.08836341008089608,
+      "grad_norm": 2.0913803577423096,
+      "learning_rate": 1.182572614107884e-06,
+      "loss": 2.4341,
+      "step": 71
+    },
+    {
+      "epoch": 0.089607965152458,
+      "grad_norm": 1.8292183876037598,
+      "learning_rate": 1.2033195020746888e-06,
+      "loss": 2.4291,
+      "step": 72
+    },
+    {
+      "epoch": 0.09085252022401992,
+      "grad_norm": 2.206770181655884,
+      "learning_rate": 1.224066390041494e-06,
+      "loss": 2.3764,
+      "step": 73
+    },
+    {
+      "epoch": 0.09209707529558182,
+      "grad_norm": 1.8263559341430664,
+      "learning_rate": 1.2448132780082988e-06,
+      "loss": 2.4082,
+      "step": 74
+    },
+    {
+      "epoch": 0.09334163036714374,
+      "grad_norm": 1.9405455589294434,
+      "learning_rate": 1.2655601659751037e-06,
+      "loss": 2.4673,
+      "step": 75
+    },
+    {
+      "epoch": 0.09458618543870566,
+      "grad_norm": 2.138108015060425,
+      "learning_rate": 1.2863070539419086e-06,
+      "loss": 2.3879,
+      "step": 76
+    },
+    {
+      "epoch": 0.09583074051026758,
+      "grad_norm": 1.9024745225906372,
+      "learning_rate": 1.307053941908714e-06,
+      "loss": 2.4344,
+      "step": 77
+    },
+    {
+      "epoch": 0.0970752955818295,
+      "grad_norm": 1.9136689901351929,
+      "learning_rate": 1.3278008298755188e-06,
+      "loss": 2.474,
+      "step": 78
+    },
+    {
+      "epoch": 0.09831985065339141,
+      "grad_norm": 2.4752795696258545,
+      "learning_rate": 1.3485477178423237e-06,
+      "loss": 2.3412,
+      "step": 79
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "grad_norm": 1.8720006942749023,
+      "learning_rate": 1.3692946058091288e-06,
+      "loss": 2.3438,
+      "step": 80
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "eval_loss": 2.3980512619018555,
+      "eval_runtime": 48.2788,
+      "eval_samples_per_second": 20.713,
+      "eval_steps_per_second": 0.87,
+      "step": 80
+    },
+    {
+      "epoch": 0.10080896079651525,
+      "grad_norm": 2.671691656112671,
+      "learning_rate": 1.3900414937759337e-06,
+      "loss": 2.3336,
+      "step": 81
+    },
+    {
+      "epoch": 0.10205351586807716,
+      "grad_norm": 2.2953391075134277,
+      "learning_rate": 1.4107883817427386e-06,
+      "loss": 2.377,
+      "step": 82
+    },
+    {
+      "epoch": 0.10329807093963908,
+      "grad_norm": 3.009018898010254,
+      "learning_rate": 1.4315352697095435e-06,
+      "loss": 2.2977,
+      "step": 83
+    },
+    {
+      "epoch": 0.104542626011201,
+      "grad_norm": 2.664454936981201,
+      "learning_rate": 1.4522821576763488e-06,
+      "loss": 2.3271,
+      "step": 84
+    },
+    {
+      "epoch": 0.10578718108276292,
+      "grad_norm": 3.017303705215454,
+      "learning_rate": 1.4730290456431537e-06,
+      "loss": 2.3251,
+      "step": 85
+    },
+    {
+      "epoch": 0.10703173615432483,
+      "grad_norm": 2.634716510772705,
+      "learning_rate": 1.4937759336099586e-06,
+      "loss": 2.332,
+      "step": 86
+    },
+    {
+      "epoch": 0.10827629122588675,
+      "grad_norm": 3.059644937515259,
+      "learning_rate": 1.5145228215767635e-06,
+      "loss": 2.3478,
+      "step": 87
+    },
+    {
+      "epoch": 0.10952084629744867,
+      "grad_norm": 2.6962637901306152,
+      "learning_rate": 1.5352697095435686e-06,
+      "loss": 2.2792,
+      "step": 88
+    },
+    {
+      "epoch": 0.11076540136901059,
+      "grad_norm": 3.419729709625244,
+      "learning_rate": 1.5560165975103735e-06,
+      "loss": 2.2571,
+      "step": 89
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "grad_norm": 2.740781545639038,
+      "learning_rate": 1.5767634854771784e-06,
+      "loss": 2.2875,
+      "step": 90
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "eval_loss": 2.30843186378479,
+      "eval_runtime": 49.9264,
+      "eval_samples_per_second": 20.029,
+      "eval_steps_per_second": 0.841,
+      "step": 90
+    },
+    {
+      "epoch": 0.1132545115121344,
+      "grad_norm": 2.5608789920806885,
+      "learning_rate": 1.5975103734439833e-06,
+      "loss": 2.2592,
+      "step": 91
+    },
+    {
+      "epoch": 0.11449906658369632,
+      "grad_norm": 2.687999963760376,
+      "learning_rate": 1.6182572614107886e-06,
+      "loss": 2.2546,
+      "step": 92
+    },
+    {
+      "epoch": 0.11574362165525824,
+      "grad_norm": 2.695909023284912,
+      "learning_rate": 1.6390041493775935e-06,
+      "loss": 2.2525,
+      "step": 93
+    },
+    {
+      "epoch": 0.11698817672682016,
+      "grad_norm": 2.818357467651367,
+      "learning_rate": 1.6597510373443984e-06,
+      "loss": 2.216,
+      "step": 94
+    },
+    {
+      "epoch": 0.11823273179838208,
+      "grad_norm": 2.884119987487793,
+      "learning_rate": 1.6804979253112035e-06,
+      "loss": 2.2321,
+      "step": 95
+    },
+    {
+      "epoch": 0.119477286869944,
+      "grad_norm": 2.52104115486145,
+      "learning_rate": 1.7012448132780084e-06,
+      "loss": 2.199,
+      "step": 96
+    },
+    {
+      "epoch": 0.12072184194150591,
+      "grad_norm": 2.420313596725464,
+      "learning_rate": 1.7219917012448133e-06,
+      "loss": 2.1862,
+      "step": 97
+    },
+    {
+      "epoch": 0.12196639701306783,
+      "grad_norm": 2.8047542572021484,
+      "learning_rate": 1.7427385892116182e-06,
+      "loss": 2.1793,
+      "step": 98
+    },
+    {
+      "epoch": 0.12321095208462975,
+      "grad_norm": 2.836482286453247,
+      "learning_rate": 1.7634854771784235e-06,
+      "loss": 2.2271,
+      "step": 99
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "grad_norm": 2.5282301902770996,
+      "learning_rate": 1.7842323651452284e-06,
+      "loss": 2.1768,
+      "step": 100
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "eval_loss": 2.2312686443328857,
+      "eval_runtime": 49.2382,
+      "eval_samples_per_second": 20.309,
+      "eval_steps_per_second": 0.853,
+      "step": 100
+    },
+    {
+      "epoch": 0.12570006222775357,
+      "grad_norm": 3.3407280445098877,
+      "learning_rate": 1.8049792531120333e-06,
+      "loss": 2.1666,
+      "step": 101
+    },
+    {
+      "epoch": 0.1269446172993155,
+      "grad_norm": 2.4754133224487305,
+      "learning_rate": 1.8257261410788382e-06,
+      "loss": 2.1768,
+      "step": 102
+    },
+    {
+      "epoch": 0.1281891723708774,
+      "grad_norm": 3.430889129638672,
+      "learning_rate": 1.8464730290456433e-06,
+      "loss": 2.1953,
+      "step": 103
+    },
+    {
+      "epoch": 0.12943372744243933,
+      "grad_norm": 2.835294246673584,
+      "learning_rate": 1.8672199170124482e-06,
+      "loss": 2.146,
+      "step": 104
+    },
+    {
+      "epoch": 0.13067828251400124,
+      "grad_norm": 3.1532323360443115,
+      "learning_rate": 1.8879668049792531e-06,
+      "loss": 2.1729,
+      "step": 105
+    },
+    {
+      "epoch": 0.13192283758556317,
+      "grad_norm": 3.2278342247009277,
+      "learning_rate": 1.908713692946058e-06,
+      "loss": 2.1336,
+      "step": 106
+    },
+    {
+      "epoch": 0.13316739265712507,
+      "grad_norm": 2.7892515659332275,
+      "learning_rate": 1.929460580912863e-06,
+      "loss": 2.141,
+      "step": 107
+    },
+    {
+      "epoch": 0.134411947728687,
+      "grad_norm": 3.0179977416992188,
+      "learning_rate": 1.9502074688796682e-06,
+      "loss": 2.1255,
+      "step": 108
+    },
+    {
+      "epoch": 0.1356565028002489,
+      "grad_norm": 2.977935552597046,
+      "learning_rate": 1.970954356846473e-06,
+      "loss": 2.0986,
+      "step": 109
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "grad_norm": 3.3042492866516113,
+      "learning_rate": 1.991701244813278e-06,
+      "loss": 2.0992,
+      "step": 110
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "eval_loss": 2.171299695968628,
+      "eval_runtime": 42.9511,
+      "eval_samples_per_second": 23.282,
+      "eval_steps_per_second": 0.978,
+      "step": 110
+    },
+    {
+      "epoch": 0.13814561294337274,
+      "grad_norm": 2.9750890731811523,
+      "learning_rate": 2.012448132780083e-06,
+      "loss": 2.1116,
+      "step": 111
+    },
+    {
+      "epoch": 0.13939016801493467,
+      "grad_norm": 2.8199286460876465,
+      "learning_rate": 2.0331950207468883e-06,
+      "loss": 2.0822,
+      "step": 112
+    },
+    {
+      "epoch": 0.14063472308649658,
+      "grad_norm": 2.930532217025757,
+      "learning_rate": 2.053941908713693e-06,
+      "loss": 2.161,
+      "step": 113
+    },
+    {
+      "epoch": 0.14187927815805848,
+      "grad_norm": 3.3400321006774902,
+      "learning_rate": 2.074688796680498e-06,
+      "loss": 2.0953,
+      "step": 114
+    },
+    {
+      "epoch": 0.1431238332296204,
+      "grad_norm": 2.716564655303955,
+      "learning_rate": 2.095435684647303e-06,
+      "loss": 2.0757,
+      "step": 115
+    },
+    {
+      "epoch": 0.14436838830118232,
+      "grad_norm": 2.7586236000061035,
+      "learning_rate": 2.116182572614108e-06,
+      "loss": 2.0979,
+      "step": 116
+    },
+    {
+      "epoch": 0.14561294337274425,
+      "grad_norm": 2.622126340866089,
+      "learning_rate": 2.136929460580913e-06,
+      "loss": 2.0748,
+      "step": 117
+    },
+    {
+      "epoch": 0.14685749844430615,
+      "grad_norm": 2.646477460861206,
+      "learning_rate": 2.157676348547718e-06,
+      "loss": 2.0605,
+      "step": 118
+    },
+    {
+      "epoch": 0.14810205351586808,
+      "grad_norm": 2.8882410526275635,
+      "learning_rate": 2.178423236514523e-06,
+      "loss": 2.0501,
+      "step": 119
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "grad_norm": 2.8302247524261475,
+      "learning_rate": 2.199170124481328e-06,
+      "loss": 2.0592,
+      "step": 120
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "eval_loss": 2.1291966438293457,
+      "eval_runtime": 47.5399,
+      "eval_samples_per_second": 21.035,
+      "eval_steps_per_second": 0.883,
+      "step": 120
+    },
+    {
+      "epoch": 0.15059116365899192,
+      "grad_norm": 3.26979660987854,
+      "learning_rate": 2.219917012448133e-06,
+      "loss": 2.0422,
+      "step": 121
+    },
+    {
+      "epoch": 0.15183571873055382,
+      "grad_norm": 2.7483913898468018,
+      "learning_rate": 2.240663900414938e-06,
+      "loss": 2.0872,
+      "step": 122
+    },
+    {
+      "epoch": 0.15308027380211575,
+      "grad_norm": 3.1252024173736572,
+      "learning_rate": 2.2614107883817427e-06,
+      "loss": 2.0684,
+      "step": 123
+    },
+    {
+      "epoch": 0.15432482887367766,
+      "grad_norm": 2.9175283908843994,
+      "learning_rate": 2.282157676348548e-06,
+      "loss": 2.0522,
+      "step": 124
+    },
+    {
+      "epoch": 0.1555693839452396,
+      "grad_norm": 2.7950754165649414,
+      "learning_rate": 2.302904564315353e-06,
+      "loss": 2.065,
+      "step": 125
+    },
+    {
+      "epoch": 0.1568139390168015,
+      "grad_norm": 2.84049654006958,
+      "learning_rate": 2.323651452282158e-06,
+      "loss": 2.0781,
+      "step": 126
+    },
+    {
+      "epoch": 0.15805849408836342,
+      "grad_norm": 2.7396647930145264,
+      "learning_rate": 2.3443983402489627e-06,
+      "loss": 2.1758,
+      "step": 127
+    },
+    {
+      "epoch": 0.15930304915992533,
+      "grad_norm": 3.0765926837921143,
+      "learning_rate": 2.365145228215768e-06,
+      "loss": 2.0172,
+      "step": 128
+    },
+    {
+      "epoch": 0.16054760423148726,
+      "grad_norm": 3.217189073562622,
+      "learning_rate": 2.385892116182573e-06,
+      "loss": 2.0286,
+      "step": 129
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "grad_norm": 3.141545295715332,
+      "learning_rate": 2.4066390041493776e-06,
+      "loss": 2.0583,
+      "step": 130
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "eval_loss": 2.0946149826049805,
+      "eval_runtime": 54.92,
+      "eval_samples_per_second": 18.208,
+      "eval_steps_per_second": 0.765,
+      "step": 130
+    },
+    {
+      "epoch": 0.16303671437461106,
+      "grad_norm": 2.850052833557129,
+      "learning_rate": 2.4273858921161828e-06,
+      "loss": 2.0746,
+      "step": 131
+    },
+    {
+      "epoch": 0.164281269446173,
+      "grad_norm": 3.28913950920105,
+      "learning_rate": 2.448132780082988e-06,
+      "loss": 2.0539,
+      "step": 132
+    },
+    {
+      "epoch": 0.1655258245177349,
+      "grad_norm": 2.8819124698638916,
+      "learning_rate": 2.468879668049793e-06,
+      "loss": 1.9843,
+      "step": 133
+    },
+    {
+      "epoch": 0.16677037958929683,
+      "grad_norm": 3.6254632472991943,
+      "learning_rate": 2.4896265560165977e-06,
+      "loss": 2.0233,
+      "step": 134
+    },
+    {
+      "epoch": 0.16801493466085873,
+      "grad_norm": 2.7385146617889404,
+      "learning_rate": 2.5103734439834028e-06,
+      "loss": 1.9667,
+      "step": 135
+    },
+    {
+      "epoch": 0.16925948973242066,
+      "grad_norm": 2.9722647666931152,
+      "learning_rate": 2.5311203319502074e-06,
+      "loss": 1.9527,
+      "step": 136
+    },
+    {
+      "epoch": 0.17050404480398257,
+      "grad_norm": 3.139526605606079,
+      "learning_rate": 2.5518672199170125e-06,
+      "loss": 1.9967,
+      "step": 137
+    },
+    {
+      "epoch": 0.1717485998755445,
+      "grad_norm": 3.399920701980591,
+      "learning_rate": 2.5726141078838172e-06,
+      "loss": 1.9798,
+      "step": 138
+    },
+    {
+      "epoch": 0.1729931549471064,
+      "grad_norm": 2.8017327785491943,
+      "learning_rate": 2.5933609958506228e-06,
+      "loss": 1.983,
+      "step": 139
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "grad_norm": 3.9967198371887207,
+      "learning_rate": 2.614107883817428e-06,
+      "loss": 1.977,
+      "step": 140
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "eval_loss": 2.059037685394287,
+      "eval_runtime": 52.6232,
+      "eval_samples_per_second": 19.003,
+      "eval_steps_per_second": 0.798,
+      "step": 140
+    },
+    {
+      "epoch": 0.17548226509023024,
+      "grad_norm": 2.893092393875122,
+      "learning_rate": 2.6348547717842326e-06,
+      "loss": 1.9888,
+      "step": 141
+    },
+    {
+      "epoch": 0.17672682016179217,
+      "grad_norm": 3.634352207183838,
+      "learning_rate": 2.6556016597510377e-06,
+      "loss": 2.0726,
+      "step": 142
+    },
+    {
+      "epoch": 0.17797137523335407,
+      "grad_norm": 3.3651444911956787,
+      "learning_rate": 2.6763485477178423e-06,
+      "loss": 1.973,
+      "step": 143
+    },
+    {
+      "epoch": 0.179215930304916,
+      "grad_norm": 3.968986988067627,
+      "learning_rate": 2.6970954356846475e-06,
+      "loss": 1.9948,
+      "step": 144
+    },
+    {
+      "epoch": 0.1804604853764779,
+      "grad_norm": 3.20105242729187,
+      "learning_rate": 2.717842323651452e-06,
+      "loss": 1.9238,
+      "step": 145
+    },
+    {
+      "epoch": 0.18170504044803984,
+      "grad_norm": 3.648339033126831,
+      "learning_rate": 2.7385892116182577e-06,
+      "loss": 1.9923,
+      "step": 146
+    },
+    {
+      "epoch": 0.18294959551960174,
+      "grad_norm": 3.0856316089630127,
+      "learning_rate": 2.7593360995850628e-06,
+      "loss": 1.9749,
+      "step": 147
+    },
+    {
+      "epoch": 0.18419415059116365,
+      "grad_norm": 3.2681071758270264,
+      "learning_rate": 2.7800829875518675e-06,
+      "loss": 1.9304,
+      "step": 148
+    },
+    {
+      "epoch": 0.18543870566272558,
+      "grad_norm": 2.634958267211914,
+      "learning_rate": 2.8008298755186726e-06,
+      "loss": 1.9237,
+      "step": 149
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "grad_norm": 2.769491672515869,
+      "learning_rate": 2.8215767634854773e-06,
+      "loss": 1.8963,
+      "step": 150
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "eval_loss": 2.0394654273986816,
+      "eval_runtime": 55.2136,
+      "eval_samples_per_second": 18.111,
+      "eval_steps_per_second": 0.761,
+      "step": 150
+    },
+    {
+      "epoch": 0.1879278158058494,
+      "grad_norm": 3.4345781803131104,
+      "learning_rate": 2.8423236514522824e-06,
+      "loss": 1.9836,
+      "step": 151
+    },
+    {
+      "epoch": 0.18917237087741132,
+      "grad_norm": 3.115727424621582,
+      "learning_rate": 2.863070539419087e-06,
+      "loss": 1.9352,
+      "step": 152
+    },
+    {
+      "epoch": 0.19041692594897325,
+      "grad_norm": 3.050652503967285,
+      "learning_rate": 2.883817427385892e-06,
+      "loss": 1.9031,
+      "step": 153
+    },
+    {
+      "epoch": 0.19166148102053515,
+      "grad_norm": 2.9404428005218506,
+      "learning_rate": 2.9045643153526977e-06,
+      "loss": 1.9196,
+      "step": 154
+    },
+    {
+      "epoch": 0.19290603609209708,
+      "grad_norm": 3.080810785293579,
+      "learning_rate": 2.9253112033195024e-06,
+      "loss": 1.9405,
+      "step": 155
+    },
+    {
+      "epoch": 0.194150591163659,
+      "grad_norm": 3.058558702468872,
+      "learning_rate": 2.9460580912863075e-06,
+      "loss": 1.9052,
+      "step": 156
+    },
+    {
+      "epoch": 0.19539514623522092,
+      "grad_norm": 3.307955026626587,
+      "learning_rate": 2.966804979253112e-06,
+      "loss": 1.9245,
+      "step": 157
+    },
+    {
+      "epoch": 0.19663970130678282,
+      "grad_norm": 2.845506191253662,
+      "learning_rate": 2.9875518672199173e-06,
+      "loss": 1.9461,
+      "step": 158
+    },
+    {
+      "epoch": 0.19788425637834475,
+      "grad_norm": 2.758654832839966,
+      "learning_rate": 3.008298755186722e-06,
+      "loss": 1.8885,
+      "step": 159
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "grad_norm": 3.161252737045288,
+      "learning_rate": 3.029045643153527e-06,
+      "loss": 1.8535,
+      "step": 160
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "eval_loss": 2.018101453781128,
+      "eval_runtime": 51.0744,
+      "eval_samples_per_second": 19.579,
+      "eval_steps_per_second": 0.822,
+      "step": 160
+    },
+    {
+      "epoch": 0.2003733665214686,
+      "grad_norm": 3.196847677230835,
+      "learning_rate": 3.0497925311203326e-06,
+      "loss": 1.8853,
+      "step": 161
+    },
+    {
+      "epoch": 0.2016179215930305,
+      "grad_norm": 2.8364663124084473,
+      "learning_rate": 3.0705394190871373e-06,
+      "loss": 1.9463,
+      "step": 162
+    },
+    {
+      "epoch": 0.2028624766645924,
+      "grad_norm": 3.0874054431915283,
+      "learning_rate": 3.0912863070539424e-06,
+      "loss": 1.9254,
+      "step": 163
+    },
+    {
+      "epoch": 0.20410703173615433,
+      "grad_norm": 2.7914493083953857,
+      "learning_rate": 3.112033195020747e-06,
+      "loss": 1.9213,
+      "step": 164
+    },
+    {
+      "epoch": 0.20535158680771623,
+      "grad_norm": 3.3871428966522217,
+      "learning_rate": 3.132780082987552e-06,
+      "loss": 1.8993,
+      "step": 165
+    },
+    {
+      "epoch": 0.20659614187927816,
+      "grad_norm": 3.096653461456299,
+      "learning_rate": 3.153526970954357e-06,
+      "loss": 1.8827,
+      "step": 166
+    },
+    {
+      "epoch": 0.20784069695084006,
+      "grad_norm": 2.8276076316833496,
+      "learning_rate": 3.174273858921162e-06,
+      "loss": 1.912,
+      "step": 167
+    },
+    {
+      "epoch": 0.209085252022402,
+      "grad_norm": 3.3058435916900635,
+      "learning_rate": 3.1950207468879666e-06,
+      "loss": 1.8381,
+      "step": 168
+    },
+    {
+      "epoch": 0.2103298070939639,
+      "grad_norm": 3.5017333030700684,
+      "learning_rate": 3.215767634854772e-06,
+      "loss": 1.9544,
+      "step": 169
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "grad_norm": 3.457296133041382,
+      "learning_rate": 3.2365145228215773e-06,
+      "loss": 1.922,
+      "step": 170
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "eval_loss": 1.988455891609192,
+      "eval_runtime": 56.2099,
+      "eval_samples_per_second": 17.79,
+      "eval_steps_per_second": 0.747,
+      "step": 170
+    },
+    {
+      "epoch": 0.21281891723708773,
+      "grad_norm": 3.1964566707611084,
+      "learning_rate": 3.257261410788382e-06,
+      "loss": 1.8925,
+      "step": 171
+    },
+    {
+      "epoch": 0.21406347230864967,
+      "grad_norm": 3.234652042388916,
+      "learning_rate": 3.278008298755187e-06,
+      "loss": 1.884,
+      "step": 172
+    },
+    {
+      "epoch": 0.21530802738021157,
+      "grad_norm": 3.14414119720459,
+      "learning_rate": 3.2987551867219918e-06,
+      "loss": 1.8905,
+      "step": 173
+    },
+    {
+      "epoch": 0.2165525824517735,
+      "grad_norm": 3.606379508972168,
+      "learning_rate": 3.319502074688797e-06,
+      "loss": 1.946,
+      "step": 174
+    },
+    {
+      "epoch": 0.2177971375233354,
+      "grad_norm": 2.986646890640259,
+      "learning_rate": 3.3402489626556016e-06,
+      "loss": 1.8691,
+      "step": 175
+    },
+    {
+      "epoch": 0.21904169259489734,
+      "grad_norm": 3.7298269271850586,
+      "learning_rate": 3.360995850622407e-06,
+      "loss": 1.8717,
+      "step": 176
+    },
+    {
+      "epoch": 0.22028624766645924,
+      "grad_norm": 3.422295093536377,
+      "learning_rate": 3.381742738589212e-06,
+      "loss": 1.8568,
+      "step": 177
+    },
+    {
+      "epoch": 0.22153080273802117,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.8471,
+      "step": 178
+    },
+    {
+      "epoch": 0.22277535780958307,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.9303,
+      "step": 179
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "grad_norm": 3.506223440170288,
+      "learning_rate": 3.423236514522822e-06,
+      "loss": 1.8237,
+      "step": 180
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "eval_loss": 1.9734643697738647,
+      "eval_runtime": 52.2815,
+      "eval_samples_per_second": 19.127,
+      "eval_steps_per_second": 0.803,
+      "step": 180
+    },
+    {
+      "epoch": 0.2252644679527069,
+      "grad_norm": 3.0156619548797607,
+      "learning_rate": 3.4439834024896267e-06,
+      "loss": 1.908,
+      "step": 181
+    },
+    {
+      "epoch": 0.2265090230242688,
+      "grad_norm": 3.55517315864563,
+      "learning_rate": 3.4647302904564318e-06,
+      "loss": 1.9104,
+      "step": 182
+    },
+    {
+      "epoch": 0.22775357809583074,
+      "grad_norm": 3.144984006881714,
+      "learning_rate": 3.4854771784232365e-06,
+      "loss": 1.8198,
+      "step": 183
+    },
+    {
+      "epoch": 0.22899813316739265,
+      "grad_norm": 4.302074432373047,
+      "learning_rate": 3.5062240663900416e-06,
+      "loss": 1.8237,
+      "step": 184
+    },
+    {
+      "epoch": 0.23024268823895458,
+      "grad_norm": 3.0522446632385254,
+      "learning_rate": 3.526970954356847e-06,
+      "loss": 1.8409,
+      "step": 185
+    },
+    {
+      "epoch": 0.23148724331051648,
+      "grad_norm": 3.8607394695281982,
+      "learning_rate": 3.5477178423236518e-06,
+      "loss": 1.8807,
+      "step": 186
+    },
+    {
+      "epoch": 0.23273179838207841,
+      "grad_norm": 2.9236302375793457,
+      "learning_rate": 3.568464730290457e-06,
+      "loss": 1.7873,
+      "step": 187
+    },
+    {
+      "epoch": 0.23397635345364032,
+      "grad_norm": 4.013780117034912,
+      "learning_rate": 3.5892116182572616e-06,
+      "loss": 1.7909,
+      "step": 188
+    },
+    {
+      "epoch": 0.23522090852520225,
+      "grad_norm": 3.0933122634887695,
+      "learning_rate": 3.6099585062240667e-06,
+      "loss": 1.8469,
+      "step": 189
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "grad_norm": 3.487816095352173,
+      "learning_rate": 3.6307053941908714e-06,
+      "loss": 1.8469,
+      "step": 190
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "eval_loss": 1.9604240655899048,
+      "eval_runtime": 53.4604,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 0.786,
+      "step": 190
+    },
+    {
+      "epoch": 0.23771001866832608,
+      "grad_norm": 2.829219341278076,
+      "learning_rate": 3.6514522821576765e-06,
+      "loss": 1.8639,
+      "step": 191
+    },
+    {
+      "epoch": 0.238954573739888,
+      "grad_norm": 3.595534324645996,
+      "learning_rate": 3.672199170124482e-06,
+      "loss": 1.8988,
+      "step": 192
+    },
+    {
+      "epoch": 0.24019912881144992,
+      "grad_norm": 2.9022483825683594,
+      "learning_rate": 3.6929460580912867e-06,
+      "loss": 1.8972,
+      "step": 193
+    },
+    {
+      "epoch": 0.24144368388301182,
+      "grad_norm": 3.0667994022369385,
+      "learning_rate": 3.713692946058092e-06,
+      "loss": 1.8499,
+      "step": 194
+    },
+    {
+      "epoch": 0.24268823895457373,
+      "grad_norm": 3.0498485565185547,
+      "learning_rate": 3.7344398340248965e-06,
+      "loss": 1.8341,
+      "step": 195
+    },
+    {
+      "epoch": 0.24393279402613566,
+      "grad_norm": 4.058084964752197,
+      "learning_rate": 3.7551867219917016e-06,
+      "loss": 1.8641,
+      "step": 196
+    },
+    {
+      "epoch": 0.24517734909769756,
+      "grad_norm": 2.9995028972625732,
+      "learning_rate": 3.7759336099585063e-06,
+      "loss": 1.7885,
+      "step": 197
+    },
+    {
+      "epoch": 0.2464219041692595,
+      "grad_norm": 3.254704475402832,
+      "learning_rate": 3.7966804979253114e-06,
+      "loss": 1.7544,
+      "step": 198
+    },
+    {
+      "epoch": 0.2476664592408214,
+      "grad_norm": 2.9126744270324707,
+      "learning_rate": 3.817427385892116e-06,
+      "loss": 1.8461,
+      "step": 199
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "grad_norm": 3.475247621536255,
+      "learning_rate": 3.838174273858922e-06,
+      "loss": 1.8095,
+      "step": 200
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "eval_loss": 1.9373760223388672,
+      "eval_runtime": 51.8733,
+      "eval_samples_per_second": 19.278,
+      "eval_steps_per_second": 0.81,
+      "step": 200
+    },
+    {
+      "epoch": 0.25015556938394523,
+      "grad_norm": 3.2663731575012207,
+      "learning_rate": 3.858921161825726e-06,
+      "loss": 1.8076,
+      "step": 201
+    },
+    {
+      "epoch": 0.25140012445550713,
+      "grad_norm": 3.1151726245880127,
+      "learning_rate": 3.879668049792531e-06,
+      "loss": 1.8143,
+      "step": 202
+    },
+    {
+      "epoch": 0.2526446795270691,
+      "grad_norm": 3.102038860321045,
+      "learning_rate": 3.9004149377593365e-06,
+      "loss": 1.8279,
+      "step": 203
+    },
+    {
+      "epoch": 0.253889234598631,
+      "grad_norm": 3.370642900466919,
+      "learning_rate": 3.921161825726142e-06,
+      "loss": 1.8522,
+      "step": 204
+    },
+    {
+      "epoch": 0.2551337896701929,
+      "grad_norm": 3.116128921508789,
+      "learning_rate": 3.941908713692946e-06,
+      "loss": 1.8104,
+      "step": 205
+    },
+    {
+      "epoch": 0.2563783447417548,
+      "grad_norm": 3.12028431892395,
+      "learning_rate": 3.962655601659751e-06,
+      "loss": 1.7723,
+      "step": 206
+    },
+    {
+      "epoch": 0.25762289981331676,
+      "grad_norm": 3.1251418590545654,
+      "learning_rate": 3.983402489626556e-06,
+      "loss": 1.863,
+      "step": 207
+    },
+    {
+      "epoch": 0.25886745488487867,
+      "grad_norm": 3.073702812194824,
+      "learning_rate": 4.004149377593361e-06,
+      "loss": 1.805,
+      "step": 208
+    },
+    {
+      "epoch": 0.26011200995644057,
+      "grad_norm": 2.9772469997406006,
+      "learning_rate": 4.024896265560166e-06,
+      "loss": 1.8251,
+      "step": 209
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "grad_norm": 2.9970037937164307,
+      "learning_rate": 4.045643153526971e-06,
+      "loss": 1.8386,
+      "step": 210
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "eval_loss": 1.9167065620422363,
+      "eval_runtime": 42.3328,
+      "eval_samples_per_second": 23.622,
+      "eval_steps_per_second": 0.992,
+      "step": 210
+    },
+    {
+      "epoch": 0.26260112009956443,
+      "grad_norm": 3.1391561031341553,
+      "learning_rate": 4.0663900414937765e-06,
+      "loss": 1.8109,
+      "step": 211
+    },
+    {
+      "epoch": 0.26384567517112634,
+      "grad_norm": 3.161538600921631,
+      "learning_rate": 4.087136929460581e-06,
+      "loss": 1.8441,
+      "step": 212
+    },
+    {
+      "epoch": 0.26509023024268824,
+      "grad_norm": 3.12412166595459,
+      "learning_rate": 4.107883817427386e-06,
+      "loss": 1.7736,
+      "step": 213
+    },
+    {
+      "epoch": 0.26633478531425014,
+      "grad_norm": 3.3241145610809326,
+      "learning_rate": 4.128630705394191e-06,
+      "loss": 1.8067,
+      "step": 214
+    },
+    {
+      "epoch": 0.26757934038581205,
+      "grad_norm": 3.503307819366455,
+      "learning_rate": 4.149377593360996e-06,
+      "loss": 1.7858,
+      "step": 215
+    },
+    {
+      "epoch": 0.268823895457374,
+      "grad_norm": 3.1009578704833984,
+      "learning_rate": 4.170124481327801e-06,
+      "loss": 1.7542,
+      "step": 216
+    },
+    {
+      "epoch": 0.2700684505289359,
+      "grad_norm": 3.2236011028289795,
+      "learning_rate": 4.190871369294606e-06,
+      "loss": 1.8006,
+      "step": 217
+    },
+    {
+      "epoch": 0.2713130056004978,
+      "grad_norm": 3.048935651779175,
+      "learning_rate": 4.211618257261411e-06,
+      "loss": 1.7706,
+      "step": 218
+    },
+    {
+      "epoch": 0.2725575606720597,
+      "grad_norm": 3.3416147232055664,
+      "learning_rate": 4.232365145228216e-06,
+      "loss": 1.8108,
+      "step": 219
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "grad_norm": 3.2010738849639893,
+      "learning_rate": 4.253112033195021e-06,
+      "loss": 1.8561,
+      "step": 220
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "eval_loss": 1.9111930131912231,
+      "eval_runtime": 50.4077,
+      "eval_samples_per_second": 19.838,
+      "eval_steps_per_second": 0.833,
+      "step": 220
+    },
+    {
+      "epoch": 0.2750466708151836,
+      "grad_norm": 3.4033374786376953,
+      "learning_rate": 4.273858921161826e-06,
+      "loss": 1.8218,
+      "step": 221
+    },
+    {
+      "epoch": 0.2762912258867455,
+      "grad_norm": 3.2086002826690674,
+      "learning_rate": 4.294605809128631e-06,
+      "loss": 1.8213,
+      "step": 222
+    },
+    {
+      "epoch": 0.2775357809583074,
+      "grad_norm": 4.090956211090088,
+      "learning_rate": 4.315352697095436e-06,
+      "loss": 1.8455,
+      "step": 223
+    },
+    {
+      "epoch": 0.27878033602986935,
+      "grad_norm": 3.1142985820770264,
+      "learning_rate": 4.336099585062241e-06,
+      "loss": 1.7737,
+      "step": 224
+    },
+    {
+      "epoch": 0.28002489110143125,
+      "grad_norm": 3.369669198989868,
+      "learning_rate": 4.356846473029046e-06,
+      "loss": 1.824,
+      "step": 225
+    },
+    {
+      "epoch": 0.28126944617299315,
+      "grad_norm": 3.165672779083252,
+      "learning_rate": 4.3775933609958506e-06,
+      "loss": 1.7768,
+      "step": 226
+    },
+    {
+      "epoch": 0.28251400124455506,
+      "grad_norm": 3.3343470096588135,
+      "learning_rate": 4.398340248962656e-06,
+      "loss": 1.7278,
+      "step": 227
+    },
+    {
+      "epoch": 0.28375855631611696,
+      "grad_norm": 3.2574994564056396,
+      "learning_rate": 4.419087136929461e-06,
+      "loss": 1.8439,
+      "step": 228
+    },
+    {
+      "epoch": 0.2850031113876789,
+      "grad_norm": 3.043928623199463,
+      "learning_rate": 4.439834024896266e-06,
+      "loss": 1.818,
+      "step": 229
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "grad_norm": 2.9703125953674316,
+      "learning_rate": 4.460580912863071e-06,
+      "loss": 1.773,
+      "step": 230
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "eval_loss": 1.898109793663025,
+      "eval_runtime": 45.6023,
+      "eval_samples_per_second": 21.929,
+      "eval_steps_per_second": 0.921,
+      "step": 230
+    },
+    {
+      "epoch": 0.2874922215308027,
+      "grad_norm": 2.9997220039367676,
+      "learning_rate": 4.481327800829876e-06,
+      "loss": 1.7727,
+      "step": 231
+    },
+    {
+      "epoch": 0.28873677660236463,
+      "grad_norm": 3.1358482837677,
+      "learning_rate": 4.502074688796681e-06,
+      "loss": 1.7764,
+      "step": 232
+    },
+    {
+      "epoch": 0.2899813316739266,
+      "grad_norm": 3.395747661590576,
+      "learning_rate": 4.5228215767634855e-06,
+      "loss": 1.7826,
+      "step": 233
+    },
+    {
+      "epoch": 0.2912258867454885,
+      "grad_norm": 3.7494754791259766,
+      "learning_rate": 4.543568464730291e-06,
+      "loss": 1.7181,
+      "step": 234
+    },
+    {
+      "epoch": 0.2924704418170504,
+      "grad_norm": 3.42293381690979,
+      "learning_rate": 4.564315352697096e-06,
+      "loss": 1.7773,
+      "step": 235
+    },
+    {
+      "epoch": 0.2937149968886123,
+      "grad_norm": 3.2524514198303223,
+      "learning_rate": 4.585062240663901e-06,
+      "loss": 1.7294,
+      "step": 236
+    },
+    {
+      "epoch": 0.29495955196017426,
+      "grad_norm": 3.657869577407837,
+      "learning_rate": 4.605809128630706e-06,
+      "loss": 1.762,
+      "step": 237
+    },
+    {
+      "epoch": 0.29620410703173616,
+      "grad_norm": 3.127372980117798,
+      "learning_rate": 4.626556016597511e-06,
+      "loss": 1.7623,
+      "step": 238
+    },
+    {
+      "epoch": 0.29744866210329807,
+      "grad_norm": 4.166962146759033,
+      "learning_rate": 4.647302904564316e-06,
+      "loss": 1.6995,
+      "step": 239
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "grad_norm": 3.094264030456543,
+      "learning_rate": 4.66804979253112e-06,
+      "loss": 1.7469,
+      "step": 240
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "eval_loss": 1.8908178806304932,
+      "eval_runtime": 44.6332,
+      "eval_samples_per_second": 22.405,
+      "eval_steps_per_second": 0.941,
+      "step": 240
+    },
+    {
+      "epoch": 0.29993777224642193,
+      "grad_norm": 3.8644745349884033,
+      "learning_rate": 4.6887966804979255e-06,
+      "loss": 1.7644,
+      "step": 241
+    },
+    {
+      "epoch": 0.30118232731798383,
+      "grad_norm": 3.1488852500915527,
+      "learning_rate": 4.709543568464731e-06,
+      "loss": 1.772,
+      "step": 242
+    },
+    {
+      "epoch": 0.30242688238954574,
+      "grad_norm": 3.3179638385772705,
+      "learning_rate": 4.730290456431536e-06,
+      "loss": 1.7588,
+      "step": 243
+    },
+    {
+      "epoch": 0.30367143746110764,
+      "grad_norm": 3.328355550765991,
+      "learning_rate": 4.751037344398341e-06,
+      "loss": 1.7417,
+      "step": 244
+    },
+    {
+      "epoch": 0.30491599253266954,
+      "grad_norm": 3.2690482139587402,
+      "learning_rate": 4.771784232365146e-06,
+      "loss": 1.7562,
+      "step": 245
+    },
+    {
+      "epoch": 0.3061605476042315,
+      "grad_norm": 3.2759454250335693,
+      "learning_rate": 4.792531120331951e-06,
+      "loss": 1.7536,
+      "step": 246
+    },
+    {
+      "epoch": 0.3074051026757934,
+      "grad_norm": 2.937964916229248,
+      "learning_rate": 4.813278008298755e-06,
+      "loss": 1.7518,
+      "step": 247
+    },
+    {
+      "epoch": 0.3086496577473553,
+      "grad_norm": 3.0995302200317383,
+      "learning_rate": 4.83402489626556e-06,
+      "loss": 1.7593,
+      "step": 248
+    },
+    {
+      "epoch": 0.3098942128189172,
+      "grad_norm": 3.230459213256836,
+      "learning_rate": 4.8547717842323655e-06,
+      "loss": 1.6962,
+      "step": 249
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "grad_norm": 3.242576837539673,
+      "learning_rate": 4.875518672199171e-06,
+      "loss": 1.7307,
+      "step": 250
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "eval_loss": 1.8805371522903442,
+      "eval_runtime": 45.9886,
+      "eval_samples_per_second": 21.745,
+      "eval_steps_per_second": 0.913,
+      "step": 250
+    },
+    {
+      "epoch": 0.3123833229620411,
+      "grad_norm": 3.097045421600342,
+      "learning_rate": 4.896265560165976e-06,
+      "loss": 1.7438,
+      "step": 251
+    },
+    {
+      "epoch": 0.313627878033603,
+      "grad_norm": 3.2428948879241943,
+      "learning_rate": 4.91701244813278e-06,
+      "loss": 1.7179,
+      "step": 252
+    },
+    {
+      "epoch": 0.3148724331051649,
+      "grad_norm": 3.196274518966675,
+      "learning_rate": 4.937759336099586e-06,
+      "loss": 1.7855,
+      "step": 253
+    },
+    {
+      "epoch": 0.31611698817672684,
+      "grad_norm": 2.978203535079956,
+      "learning_rate": 4.95850622406639e-06,
+      "loss": 1.7144,
+      "step": 254
+    },
+    {
+      "epoch": 0.31736154324828875,
+      "grad_norm": 3.2641701698303223,
+      "learning_rate": 4.979253112033195e-06,
+      "loss": 1.7427,
+      "step": 255
+    },
+    {
+      "epoch": 0.31860609831985065,
+      "grad_norm": 2.7441232204437256,
+      "learning_rate": 5e-06,
+      "loss": 1.7619,
+      "step": 256
+    },
+    {
+      "epoch": 0.31985065339141255,
+      "grad_norm": 3.0723495483398438,
+      "learning_rate": 4.99769372693727e-06,
+      "loss": 1.8017,
+      "step": 257
+    },
+    {
+      "epoch": 0.3210952084629745,
+      "grad_norm": 3.104752540588379,
+      "learning_rate": 4.995387453874539e-06,
+      "loss": 1.7761,
+      "step": 258
+    },
+    {
+      "epoch": 0.3223397635345364,
+      "grad_norm": 3.138627052307129,
+      "learning_rate": 4.993081180811809e-06,
+      "loss": 1.7879,
+      "step": 259
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "grad_norm": 2.8567333221435547,
+      "learning_rate": 4.990774907749078e-06,
+      "loss": 1.7561,
+      "step": 260
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "eval_loss": 1.8606494665145874,
+      "eval_runtime": 46.453,
+      "eval_samples_per_second": 21.527,
+      "eval_steps_per_second": 0.904,
+      "step": 260
+    },
+    {
+      "epoch": 0.3248288736776602,
+      "grad_norm": 3.2696540355682373,
+      "learning_rate": 4.988468634686347e-06,
+      "loss": 1.7201,
+      "step": 261
+    },
+    {
+      "epoch": 0.3260734287492221,
+      "grad_norm": 2.6692731380462646,
+      "learning_rate": 4.986162361623617e-06,
+      "loss": 1.7264,
+      "step": 262
+    },
+    {
+      "epoch": 0.3273179838207841,
+      "grad_norm": 3.047549247741699,
+      "learning_rate": 4.983856088560886e-06,
+      "loss": 1.7362,
+      "step": 263
+    },
+    {
+      "epoch": 0.328562538892346,
+      "grad_norm": 3.0667457580566406,
+      "learning_rate": 4.981549815498156e-06,
+      "loss": 1.7321,
+      "step": 264
+    },
+    {
+      "epoch": 0.3298070939639079,
+      "grad_norm": 2.560047149658203,
+      "learning_rate": 4.979243542435424e-06,
+      "loss": 1.7508,
+      "step": 265
+    },
+    {
+      "epoch": 0.3310516490354698,
+      "grad_norm": 3.26595401763916,
+      "learning_rate": 4.976937269372694e-06,
+      "loss": 1.7248,
+      "step": 266
+    },
+    {
+      "epoch": 0.33229620410703176,
+      "grad_norm": 2.929210662841797,
+      "learning_rate": 4.974630996309964e-06,
+      "loss": 1.697,
+      "step": 267
+    },
+    {
+      "epoch": 0.33354075917859366,
+      "grad_norm": 3.1526286602020264,
+      "learning_rate": 4.972324723247233e-06,
+      "loss": 1.7385,
+      "step": 268
+    },
+    {
+      "epoch": 0.33478531425015556,
+      "grad_norm": 3.001619577407837,
+      "learning_rate": 4.970018450184502e-06,
+      "loss": 1.7072,
+      "step": 269
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "grad_norm": 3.2464189529418945,
+      "learning_rate": 4.9677121771217715e-06,
+      "loss": 1.7361,
+      "step": 270
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "eval_loss": 1.860226035118103,
+      "eval_runtime": 46.6881,
+      "eval_samples_per_second": 21.419,
+      "eval_steps_per_second": 0.9,
+      "step": 270
+    },
+    {
+      "epoch": 0.3372744243932794,
+      "grad_norm": 3.2104530334472656,
+      "learning_rate": 4.965405904059041e-06,
+      "loss": 1.7171,
+      "step": 271
+    },
+    {
+      "epoch": 0.33851897946484133,
+      "grad_norm": 3.146847724914551,
+      "learning_rate": 4.96309963099631e-06,
+      "loss": 1.7311,
+      "step": 272
+    },
+    {
+      "epoch": 0.33976353453640323,
+      "grad_norm": 3.1431286334991455,
+      "learning_rate": 4.96079335793358e-06,
+      "loss": 1.7345,
+      "step": 273
+    },
+    {
+      "epoch": 0.34100808960796514,
+      "grad_norm": 2.847163677215576,
+      "learning_rate": 4.958487084870849e-06,
+      "loss": 1.7262,
+      "step": 274
+    },
+    {
+      "epoch": 0.3422526446795271,
+      "grad_norm": 3.1987810134887695,
+      "learning_rate": 4.956180811808119e-06,
+      "loss": 1.7945,
+      "step": 275
+    },
+    {
+      "epoch": 0.343497199751089,
+      "grad_norm": 2.7475385665893555,
+      "learning_rate": 4.953874538745388e-06,
+      "loss": 1.76,
+      "step": 276
+    },
+    {
+      "epoch": 0.3447417548226509,
+      "grad_norm": 3.0427663326263428,
+      "learning_rate": 4.9515682656826574e-06,
+      "loss": 1.7324,
+      "step": 277
+    },
+    {
+      "epoch": 0.3459863098942128,
+      "grad_norm": 3.605212688446045,
+      "learning_rate": 4.949261992619927e-06,
+      "loss": 1.7712,
+      "step": 278
+    },
+    {
+      "epoch": 0.3472308649657747,
+      "grad_norm": 3.0564935207366943,
+      "learning_rate": 4.946955719557196e-06,
+      "loss": 1.6792,
+      "step": 279
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "grad_norm": 3.1069741249084473,
+      "learning_rate": 4.944649446494466e-06,
+      "loss": 1.7145,
+      "step": 280
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "eval_loss": 1.8537051677703857,
+      "eval_runtime": 47.4813,
+      "eval_samples_per_second": 21.061,
+      "eval_steps_per_second": 0.885,
+      "step": 280
+    },
+    {
+      "epoch": 0.3497199751088986,
+      "grad_norm": 2.8021512031555176,
+      "learning_rate": 4.942343173431734e-06,
+      "loss": 1.7145,
+      "step": 281
+    },
+    {
+      "epoch": 0.3509645301804605,
+      "grad_norm": 3.5549023151397705,
+      "learning_rate": 4.940036900369004e-06,
+      "loss": 1.7642,
+      "step": 282
+    },
+    {
+      "epoch": 0.3522090852520224,
+      "grad_norm": 2.7648985385894775,
+      "learning_rate": 4.937730627306274e-06,
+      "loss": 1.7255,
+      "step": 283
+    },
+    {
+      "epoch": 0.35345364032358434,
+      "grad_norm": 3.0815863609313965,
+      "learning_rate": 4.9354243542435426e-06,
+      "loss": 1.7055,
+      "step": 284
+    },
+    {
+      "epoch": 0.35469819539514624,
+      "grad_norm": 2.9009227752685547,
+      "learning_rate": 4.933118081180812e-06,
+      "loss": 1.7019,
+      "step": 285
+    },
+    {
+      "epoch": 0.35594275046670815,
+      "grad_norm": 2.618429660797119,
+      "learning_rate": 4.930811808118081e-06,
+      "loss": 1.6318,
+      "step": 286
+    },
+    {
+      "epoch": 0.35718730553827005,
+      "grad_norm": 3.368230104446411,
+      "learning_rate": 4.928505535055351e-06,
+      "loss": 1.7304,
+      "step": 287
+    },
+    {
+      "epoch": 0.358431860609832,
+      "grad_norm": 3.256889820098877,
+      "learning_rate": 4.92619926199262e-06,
+      "loss": 1.7019,
+      "step": 288
+    },
+    {
+      "epoch": 0.3596764156813939,
+      "grad_norm": 2.9366304874420166,
+      "learning_rate": 4.92389298892989e-06,
+      "loss": 1.734,
+      "step": 289
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "grad_norm": 3.0393142700195312,
+      "learning_rate": 4.921586715867159e-06,
+      "loss": 1.7504,
+      "step": 290
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "eval_loss": 1.8359886407852173,
+      "eval_runtime": 49.7622,
+      "eval_samples_per_second": 20.096,
+      "eval_steps_per_second": 0.844,
+      "step": 290
+    },
+    {
+      "epoch": 0.3621655258245177,
+      "grad_norm": 2.7288589477539062,
+      "learning_rate": 4.9192804428044285e-06,
+      "loss": 1.7004,
+      "step": 291
+    },
+    {
+      "epoch": 0.3634100808960797,
+      "grad_norm": 3.2793378829956055,
+      "learning_rate": 4.916974169741698e-06,
+      "loss": 1.7028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3646546359676416,
+      "grad_norm": 2.960880756378174,
+      "learning_rate": 4.914667896678967e-06,
+      "loss": 1.6759,
+      "step": 293
+    },
+    {
+      "epoch": 0.3658991910392035,
+      "grad_norm": 2.836421012878418,
+      "learning_rate": 4.912361623616237e-06,
+      "loss": 1.6475,
+      "step": 294
+    },
+    {
+      "epoch": 0.3671437461107654,
+      "grad_norm": 2.9495465755462646,
+      "learning_rate": 4.910055350553506e-06,
+      "loss": 1.7076,
+      "step": 295
+    },
+    {
+      "epoch": 0.3683883011823273,
+      "grad_norm": 2.954730272293091,
+      "learning_rate": 4.907749077490776e-06,
+      "loss": 1.7261,
+      "step": 296
+    },
+    {
+      "epoch": 0.36963285625388925,
+      "grad_norm": 2.8167543411254883,
+      "learning_rate": 4.905442804428044e-06,
+      "loss": 1.6939,
+      "step": 297
+    },
+    {
+      "epoch": 0.37087741132545116,
+      "grad_norm": 2.857316017150879,
+      "learning_rate": 4.903136531365314e-06,
+      "loss": 1.7362,
+      "step": 298
+    },
+    {
+      "epoch": 0.37212196639701306,
+      "grad_norm": 2.8612918853759766,
+      "learning_rate": 4.900830258302584e-06,
+      "loss": 1.6489,
+      "step": 299
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "grad_norm": 2.9618914127349854,
+      "learning_rate": 4.898523985239853e-06,
+      "loss": 1.718,
+      "step": 300
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "eval_loss": 1.827571988105774,
+      "eval_runtime": 46.4552,
+      "eval_samples_per_second": 21.526,
+      "eval_steps_per_second": 0.904,
+      "step": 300
+    },
+    {
+      "epoch": 0.3746110765401369,
+      "grad_norm": 2.8706953525543213,
+      "learning_rate": 4.896217712177122e-06,
+      "loss": 1.6925,
+      "step": 301
+    },
+    {
+      "epoch": 0.3758556316116988,
+      "grad_norm": 3.2565090656280518,
+      "learning_rate": 4.893911439114391e-06,
+      "loss": 1.7369,
+      "step": 302
+    },
+    {
+      "epoch": 0.37710018668326073,
+      "grad_norm": 2.928858995437622,
+      "learning_rate": 4.891605166051661e-06,
+      "loss": 1.6882,
+      "step": 303
+    },
+    {
+      "epoch": 0.37834474175482263,
+      "grad_norm": 3.042314052581787,
+      "learning_rate": 4.88929889298893e-06,
+      "loss": 1.6466,
+      "step": 304
+    },
+    {
+      "epoch": 0.3795892968263846,
+      "grad_norm": 2.7509143352508545,
+      "learning_rate": 4.8869926199262e-06,
+      "loss": 1.685,
+      "step": 305
+    },
+    {
+      "epoch": 0.3808338518979465,
+      "grad_norm": 2.9601588249206543,
+      "learning_rate": 4.884686346863469e-06,
+      "loss": 1.6772,
+      "step": 306
+    },
+    {
+      "epoch": 0.3820784069695084,
+      "grad_norm": 2.618608236312866,
+      "learning_rate": 4.8823800738007384e-06,
+      "loss": 1.7127,
+      "step": 307
+    },
+    {
+      "epoch": 0.3833229620410703,
+      "grad_norm": 3.047581195831299,
+      "learning_rate": 4.880073800738008e-06,
+      "loss": 1.7174,
+      "step": 308
+    },
+    {
+      "epoch": 0.3845675171126322,
+      "grad_norm": 2.87735652923584,
+      "learning_rate": 4.877767527675277e-06,
+      "loss": 1.6677,
+      "step": 309
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "grad_norm": 2.7169721126556396,
+      "learning_rate": 4.875461254612546e-06,
+      "loss": 1.7486,
+      "step": 310
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "eval_loss": 1.8221794366836548,
+      "eval_runtime": 43.1681,
+      "eval_samples_per_second": 23.165,
+      "eval_steps_per_second": 0.973,
+      "step": 310
+    },
+    {
+      "epoch": 0.38705662725575607,
+      "grad_norm": 2.7094991207122803,
+      "learning_rate": 4.873154981549816e-06,
+      "loss": 1.7348,
+      "step": 311
+    },
+    {
+      "epoch": 0.388301182327318,
+      "grad_norm": 2.8989078998565674,
+      "learning_rate": 4.8708487084870856e-06,
+      "loss": 1.7352,
+      "step": 312
+    },
+    {
+      "epoch": 0.3895457373988799,
+      "grad_norm": 2.9565601348876953,
+      "learning_rate": 4.868542435424355e-06,
+      "loss": 1.7063,
+      "step": 313
+    },
+    {
+      "epoch": 0.39079029247044184,
+      "grad_norm": 2.8354082107543945,
+      "learning_rate": 4.8662361623616235e-06,
+      "loss": 1.6808,
+      "step": 314
+    },
+    {
+      "epoch": 0.39203484754200374,
+      "grad_norm": 2.8287479877471924,
+      "learning_rate": 4.863929889298894e-06,
+      "loss": 1.6731,
+      "step": 315
+    },
+    {
+      "epoch": 0.39327940261356564,
+      "grad_norm": 2.6537821292877197,
+      "learning_rate": 4.861623616236163e-06,
+      "loss": 1.695,
+      "step": 316
+    },
+    {
+      "epoch": 0.39452395768512755,
+      "grad_norm": 2.7135190963745117,
+      "learning_rate": 4.859317343173432e-06,
+      "loss": 1.6897,
+      "step": 317
+    },
+    {
+      "epoch": 0.3957685127566895,
+      "grad_norm": 3.1019129753112793,
+      "learning_rate": 4.857011070110701e-06,
+      "loss": 1.7219,
+      "step": 318
+    },
+    {
+      "epoch": 0.3970130678282514,
+      "grad_norm": 2.945737838745117,
+      "learning_rate": 4.854704797047971e-06,
+      "loss": 1.7316,
+      "step": 319
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "grad_norm": 2.7668251991271973,
+      "learning_rate": 4.85239852398524e-06,
+      "loss": 1.6461,
+      "step": 320
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "eval_loss": 1.8210570812225342,
+      "eval_runtime": 46.2552,
+      "eval_samples_per_second": 21.619,
+      "eval_steps_per_second": 0.908,
+      "step": 320
+    },
+    {
+      "epoch": 0.3995021779713752,
+      "grad_norm": 2.738109588623047,
+      "learning_rate": 4.8500922509225095e-06,
+      "loss": 1.6907,
+      "step": 321
+    },
+    {
+      "epoch": 0.4007467330429372,
+      "grad_norm": 3.0195062160491943,
+      "learning_rate": 4.847785977859779e-06,
+      "loss": 1.7099,
+      "step": 322
+    },
+    {
+      "epoch": 0.4019912881144991,
+      "grad_norm": 2.8672404289245605,
+      "learning_rate": 4.845479704797048e-06,
+      "loss": 1.6509,
+      "step": 323
+    },
+    {
+      "epoch": 0.403235843186061,
+      "grad_norm": 2.9041008949279785,
+      "learning_rate": 4.843173431734318e-06,
+      "loss": 1.5818,
+      "step": 324
+    },
+    {
+      "epoch": 0.4044803982576229,
+      "grad_norm": 2.9105985164642334,
+      "learning_rate": 4.840867158671587e-06,
+      "loss": 1.7002,
+      "step": 325
+    },
+    {
+      "epoch": 0.4057249533291848,
+      "grad_norm": 2.749009847640991,
+      "learning_rate": 4.838560885608857e-06,
+      "loss": 1.7021,
+      "step": 326
+    },
+    {
+      "epoch": 0.40696950840074675,
+      "grad_norm": 2.6998822689056396,
+      "learning_rate": 4.836254612546126e-06,
+      "loss": 1.6853,
+      "step": 327
+    },
+    {
+      "epoch": 0.40821406347230865,
+      "grad_norm": 2.678201675415039,
+      "learning_rate": 4.8339483394833955e-06,
+      "loss": 1.6934,
+      "step": 328
+    },
+    {
+      "epoch": 0.40945861854387056,
+      "grad_norm": 2.774845600128174,
+      "learning_rate": 4.831642066420665e-06,
+      "loss": 1.657,
+      "step": 329
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "grad_norm": 2.6363000869750977,
+      "learning_rate": 4.8293357933579335e-06,
+      "loss": 1.6555,
+      "step": 330
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "eval_loss": 1.8126581907272339,
+      "eval_runtime": 52.7717,
+      "eval_samples_per_second": 18.95,
+      "eval_steps_per_second": 0.796,
+      "step": 330
+    },
+    {
+      "epoch": 0.4119477286869944,
+      "grad_norm": 3.011659622192383,
+      "learning_rate": 4.827029520295204e-06,
+      "loss": 1.7202,
+      "step": 331
+    },
+    {
+      "epoch": 0.4131922837585563,
+      "grad_norm": 2.8322594165802,
+      "learning_rate": 4.824723247232473e-06,
+      "loss": 1.6736,
+      "step": 332
+    },
+    {
+      "epoch": 0.4144368388301182,
+      "grad_norm": 2.9360146522521973,
+      "learning_rate": 4.822416974169742e-06,
+      "loss": 1.7213,
+      "step": 333
+    },
+    {
+      "epoch": 0.41568139390168013,
+      "grad_norm": 2.810206413269043,
+      "learning_rate": 4.820110701107011e-06,
+      "loss": 1.6867,
+      "step": 334
+    },
+    {
+      "epoch": 0.4169259489732421,
+      "grad_norm": 2.6864097118377686,
+      "learning_rate": 4.817804428044281e-06,
+      "loss": 1.6854,
+      "step": 335
+    },
+    {
+      "epoch": 0.418170504044804,
+      "grad_norm": 2.7396061420440674,
+      "learning_rate": 4.81549815498155e-06,
+      "loss": 1.68,
+      "step": 336
+    },
+    {
+      "epoch": 0.4194150591163659,
+      "grad_norm": 2.602771520614624,
+      "learning_rate": 4.8131918819188194e-06,
+      "loss": 1.6269,
+      "step": 337
+    },
+    {
+      "epoch": 0.4206596141879278,
+      "grad_norm": 2.8174757957458496,
+      "learning_rate": 4.810885608856089e-06,
+      "loss": 1.6981,
+      "step": 338
+    },
+    {
+      "epoch": 0.42190416925948976,
+      "grad_norm": 3.2398130893707275,
+      "learning_rate": 4.808579335793358e-06,
+      "loss": 1.7061,
+      "step": 339
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "grad_norm": 2.7119736671447754,
+      "learning_rate": 4.806273062730628e-06,
+      "loss": 1.6875,
+      "step": 340
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "eval_loss": 1.7973068952560425,
+      "eval_runtime": 53.5791,
+      "eval_samples_per_second": 18.664,
+      "eval_steps_per_second": 0.784,
+      "step": 340
+    },
+    {
+      "epoch": 0.42439327940261357,
+      "grad_norm": 2.868533134460449,
+      "learning_rate": 4.803966789667897e-06,
+      "loss": 1.6967,
+      "step": 341
+    },
+    {
+      "epoch": 0.42563783447417547,
+      "grad_norm": 2.528083562850952,
+      "learning_rate": 4.8016605166051665e-06,
+      "loss": 1.6195,
+      "step": 342
+    },
+    {
+      "epoch": 0.4268823895457374,
+      "grad_norm": 2.7885115146636963,
+      "learning_rate": 4.799354243542436e-06,
+      "loss": 1.6197,
+      "step": 343
+    },
+    {
+      "epoch": 0.42812694461729933,
+      "grad_norm": 2.768247604370117,
+      "learning_rate": 4.797047970479705e-06,
+      "loss": 1.6271,
+      "step": 344
+    },
+    {
+      "epoch": 0.42937149968886124,
+      "grad_norm": 2.7710275650024414,
+      "learning_rate": 4.794741697416975e-06,
+      "loss": 1.6768,
+      "step": 345
+    },
+    {
+      "epoch": 0.43061605476042314,
+      "grad_norm": 2.968337297439575,
+      "learning_rate": 4.792435424354243e-06,
+      "loss": 1.6433,
+      "step": 346
+    },
+    {
+      "epoch": 0.43186060983198504,
+      "grad_norm": 2.5739808082580566,
+      "learning_rate": 4.790129151291514e-06,
+      "loss": 1.6327,
+      "step": 347
+    },
+    {
+      "epoch": 0.433105164903547,
+      "grad_norm": 2.7676408290863037,
+      "learning_rate": 4.787822878228783e-06,
+      "loss": 1.6178,
+      "step": 348
+    },
+    {
+      "epoch": 0.4343497199751089,
+      "grad_norm": 2.959059476852417,
+      "learning_rate": 4.7855166051660525e-06,
+      "loss": 1.6809,
+      "step": 349
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "grad_norm": 2.559967279434204,
+      "learning_rate": 4.783210332103321e-06,
+      "loss": 1.6704,
+      "step": 350
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "eval_loss": 1.8013949394226074,
+      "eval_runtime": 50.4649,
+      "eval_samples_per_second": 19.816,
+      "eval_steps_per_second": 0.832,
+      "step": 350
+    },
+    {
+      "epoch": 0.4368388301182327,
+      "grad_norm": 2.817901611328125,
+      "learning_rate": 4.7809040590405905e-06,
+      "loss": 1.6296,
+      "step": 351
+    },
+    {
+      "epoch": 0.43808338518979467,
+      "grad_norm": 2.7116055488586426,
+      "learning_rate": 4.778597785977861e-06,
+      "loss": 1.6652,
+      "step": 352
+    },
+    {
+      "epoch": 0.4393279402613566,
+      "grad_norm": 2.544750452041626,
+      "learning_rate": 4.776291512915129e-06,
+      "loss": 1.5668,
+      "step": 353
+    },
+    {
+      "epoch": 0.4405724953329185,
+      "grad_norm": 2.720534086227417,
+      "learning_rate": 4.773985239852399e-06,
+      "loss": 1.6152,
+      "step": 354
+    },
+    {
+      "epoch": 0.4418170504044804,
+      "grad_norm": 2.7270126342773438,
+      "learning_rate": 4.771678966789668e-06,
+      "loss": 1.5624,
+      "step": 355
+    },
+    {
+      "epoch": 0.44306160547604234,
+      "grad_norm": 2.7986624240875244,
+      "learning_rate": 4.769372693726938e-06,
+      "loss": 1.6621,
+      "step": 356
+    },
+    {
+      "epoch": 0.44430616054760425,
+      "grad_norm": 2.943107843399048,
+      "learning_rate": 4.767066420664207e-06,
+      "loss": 1.6713,
+      "step": 357
+    },
+    {
+      "epoch": 0.44555071561916615,
+      "grad_norm": 2.739898681640625,
+      "learning_rate": 4.7647601476014765e-06,
+      "loss": 1.6363,
+      "step": 358
+    },
+    {
+      "epoch": 0.44679527069072805,
+      "grad_norm": 2.8729329109191895,
+      "learning_rate": 4.762453874538746e-06,
+      "loss": 1.6605,
+      "step": 359
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "grad_norm": 2.909029483795166,
+      "learning_rate": 4.760147601476015e-06,
+      "loss": 1.6277,
+      "step": 360
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "eval_loss": 1.8110179901123047,
+      "eval_runtime": 47.924,
+      "eval_samples_per_second": 20.866,
+      "eval_steps_per_second": 0.876,
+      "step": 360
+    },
+    {
+      "epoch": 0.4492843808338519,
+      "grad_norm": 3.3826682567596436,
+      "learning_rate": 4.757841328413285e-06,
+      "loss": 1.7155,
+      "step": 361
+    },
+    {
+      "epoch": 0.4505289359054138,
+      "grad_norm": 3.0586729049682617,
+      "learning_rate": 4.755535055350554e-06,
+      "loss": 1.5991,
+      "step": 362
+    },
+    {
+      "epoch": 0.4517734909769757,
+      "grad_norm": 2.645576238632202,
+      "learning_rate": 4.753228782287823e-06,
+      "loss": 1.6504,
+      "step": 363
+    },
+    {
+      "epoch": 0.4530180460485376,
+      "grad_norm": 2.705967664718628,
+      "learning_rate": 4.750922509225093e-06,
+      "loss": 1.6761,
+      "step": 364
+    },
+    {
+      "epoch": 0.4542626011200996,
+      "grad_norm": 2.7651922702789307,
+      "learning_rate": 4.748616236162362e-06,
+      "loss": 1.61,
+      "step": 365
+    },
+    {
+      "epoch": 0.4555071561916615,
+      "grad_norm": 2.5654966831207275,
+      "learning_rate": 4.746309963099631e-06,
+      "loss": 1.6335,
+      "step": 366
+    },
+    {
+      "epoch": 0.4567517112632234,
+      "grad_norm": 2.6606173515319824,
+      "learning_rate": 4.7440036900369e-06,
+      "loss": 1.6207,
+      "step": 367
+    },
+    {
+      "epoch": 0.4579962663347853,
+      "grad_norm": 2.724755048751831,
+      "learning_rate": 4.741697416974171e-06,
+      "loss": 1.6329,
+      "step": 368
+    },
+    {
+      "epoch": 0.45924082140634725,
+      "grad_norm": 2.7019615173339844,
+      "learning_rate": 4.739391143911439e-06,
+      "loss": 1.6763,
+      "step": 369
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "grad_norm": 2.6594254970550537,
+      "learning_rate": 4.737084870848709e-06,
+      "loss": 1.6174,
+      "step": 370
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "eval_loss": 1.7924141883850098,
+      "eval_runtime": 53.0349,
+      "eval_samples_per_second": 18.855,
+      "eval_steps_per_second": 0.792,
+      "step": 370
+    },
+    {
+      "epoch": 0.46172993154947106,
+      "grad_norm": 2.668421745300293,
+      "learning_rate": 4.734778597785978e-06,
+      "loss": 1.6421,
+      "step": 371
+    },
+    {
+      "epoch": 0.46297448662103297,
+      "grad_norm": 2.7148995399475098,
+      "learning_rate": 4.7324723247232475e-06,
+      "loss": 1.6635,
+      "step": 372
+    },
+    {
+      "epoch": 0.4642190416925949,
+      "grad_norm": 3.156646490097046,
+      "learning_rate": 4.730166051660517e-06,
+      "loss": 1.6679,
+      "step": 373
+    },
+    {
+      "epoch": 0.46546359676415683,
+      "grad_norm": 2.8170645236968994,
+      "learning_rate": 4.727859778597786e-06,
+      "loss": 1.6492,
+      "step": 374
+    },
+    {
+      "epoch": 0.46670815183571873,
+      "grad_norm": 2.9796457290649414,
+      "learning_rate": 4.725553505535056e-06,
+      "loss": 1.6025,
+      "step": 375
+    },
+    {
+      "epoch": 0.46795270690728064,
+      "grad_norm": 2.6493406295776367,
+      "learning_rate": 4.723247232472325e-06,
+      "loss": 1.6398,
+      "step": 376
+    },
+    {
+      "epoch": 0.46919726197884254,
+      "grad_norm": 3.2643542289733887,
+      "learning_rate": 4.720940959409595e-06,
+      "loss": 1.6413,
+      "step": 377
+    },
+    {
+      "epoch": 0.4704418170504045,
+      "grad_norm": 2.7613179683685303,
+      "learning_rate": 4.718634686346864e-06,
+      "loss": 1.6251,
+      "step": 378
+    },
+    {
+      "epoch": 0.4716863721219664,
+      "grad_norm": 3.1216518878936768,
+      "learning_rate": 4.716328413284133e-06,
+      "loss": 1.6268,
+      "step": 379
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "grad_norm": 2.571647882461548,
+      "learning_rate": 4.714022140221403e-06,
+      "loss": 1.6245,
+      "step": 380
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "eval_loss": 1.7875893115997314,
+      "eval_runtime": 49.3058,
+      "eval_samples_per_second": 20.282,
+      "eval_steps_per_second": 0.852,
+      "step": 380
+    },
+    {
+      "epoch": 0.4741754822650902,
+      "grad_norm": 3.443906545639038,
+      "learning_rate": 4.711715867158672e-06,
+      "loss": 1.6799,
+      "step": 381
+    },
+    {
+      "epoch": 0.47542003733665217,
+      "grad_norm": 2.7887957096099854,
+      "learning_rate": 4.709409594095941e-06,
+      "loss": 1.6589,
+      "step": 382
+    },
+    {
+      "epoch": 0.47666459240821407,
+      "grad_norm": 3.2833151817321777,
+      "learning_rate": 4.70710332103321e-06,
+      "loss": 1.6064,
+      "step": 383
+    },
+    {
+      "epoch": 0.477909147479776,
+      "grad_norm": 2.6975057125091553,
+      "learning_rate": 4.704797047970481e-06,
+      "loss": 1.6029,
+      "step": 384
+    },
+    {
+      "epoch": 0.4791537025513379,
+      "grad_norm": 2.745246171951294,
+      "learning_rate": 4.702490774907749e-06,
+      "loss": 1.7127,
+      "step": 385
+    },
+    {
+      "epoch": 0.48039825762289984,
+      "grad_norm": 3.0522775650024414,
+      "learning_rate": 4.700184501845019e-06,
+      "loss": 1.6508,
+      "step": 386
+    },
+    {
+      "epoch": 0.48164281269446174,
+      "grad_norm": 2.689218044281006,
+      "learning_rate": 4.697878228782288e-06,
+      "loss": 1.6971,
+      "step": 387
+    },
+    {
+      "epoch": 0.48288736776602365,
+      "grad_norm": 3.060361862182617,
+      "learning_rate": 4.6955719557195575e-06,
+      "loss": 1.6464,
+      "step": 388
+    },
+    {
+      "epoch": 0.48413192283758555,
+      "grad_norm": 2.872262477874756,
+      "learning_rate": 4.693265682656827e-06,
+      "loss": 1.6293,
+      "step": 389
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "grad_norm": 2.980050563812256,
+      "learning_rate": 4.690959409594096e-06,
+      "loss": 1.6435,
+      "step": 390
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "eval_loss": 1.7791496515274048,
+      "eval_runtime": 50.0299,
+      "eval_samples_per_second": 19.988,
+      "eval_steps_per_second": 0.839,
+      "step": 390
+    },
+    {
+      "epoch": 0.4866210329807094,
+      "grad_norm": 2.764827251434326,
+      "learning_rate": 4.688653136531366e-06,
+      "loss": 1.6279,
+      "step": 391
+    },
+    {
+      "epoch": 0.4878655880522713,
+      "grad_norm": 2.6912333965301514,
+      "learning_rate": 4.686346863468635e-06,
+      "loss": 1.7276,
+      "step": 392
+    },
+    {
+      "epoch": 0.4891101431238332,
+      "grad_norm": 2.8358330726623535,
+      "learning_rate": 4.6840405904059046e-06,
+      "loss": 1.6482,
+      "step": 393
+    },
+    {
+      "epoch": 0.4903546981953951,
+      "grad_norm": 2.6581575870513916,
+      "learning_rate": 4.681734317343174e-06,
+      "loss": 1.6853,
+      "step": 394
+    },
+    {
+      "epoch": 0.4915992532669571,
+      "grad_norm": 2.941425323486328,
+      "learning_rate": 4.6794280442804426e-06,
+      "loss": 1.6075,
+      "step": 395
+    },
+    {
+      "epoch": 0.492843808338519,
+      "grad_norm": 3.2334814071655273,
+      "learning_rate": 4.677121771217713e-06,
+      "loss": 1.6157,
+      "step": 396
+    },
+    {
+      "epoch": 0.4940883634100809,
+      "grad_norm": 2.984410524368286,
+      "learning_rate": 4.674815498154982e-06,
+      "loss": 1.6257,
+      "step": 397
+    },
+    {
+      "epoch": 0.4953329184816428,
+      "grad_norm": 3.0650789737701416,
+      "learning_rate": 4.672509225092252e-06,
+      "loss": 1.6573,
+      "step": 398
+    },
+    {
+      "epoch": 0.49657747355320475,
+      "grad_norm": 2.798664093017578,
+      "learning_rate": 4.67020295202952e-06,
+      "loss": 1.638,
+      "step": 399
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "grad_norm": 2.8266706466674805,
+      "learning_rate": 4.66789667896679e-06,
+      "loss": 1.6484,
+      "step": 400
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "eval_loss": 1.7714862823486328,
+      "eval_runtime": 48.0657,
+      "eval_samples_per_second": 20.805,
+      "eval_steps_per_second": 0.874,
+      "step": 400
+    },
+    {
+      "epoch": 0.49906658369632856,
+      "grad_norm": 2.9928388595581055,
+      "learning_rate": 4.66559040590406e-06,
+      "loss": 1.6721,
+      "step": 401
+    },
+    {
+      "epoch": 0.5003111387678905,
+      "grad_norm": 2.573514461517334,
+      "learning_rate": 4.6632841328413285e-06,
+      "loss": 1.679,
+      "step": 402
+    },
+    {
+      "epoch": 0.5015556938394524,
+      "grad_norm": 3.1736996173858643,
+      "learning_rate": 4.660977859778598e-06,
+      "loss": 1.7124,
+      "step": 403
+    },
+    {
+      "epoch": 0.5028002489110143,
+      "grad_norm": 2.858849287033081,
+      "learning_rate": 4.658671586715867e-06,
+      "loss": 1.6622,
+      "step": 404
+    },
+    {
+      "epoch": 0.5040448039825762,
+      "grad_norm": 2.825698137283325,
+      "learning_rate": 4.656365313653137e-06,
+      "loss": 1.6464,
+      "step": 405
+    },
+    {
+      "epoch": 0.5052893590541382,
+      "grad_norm": 3.783891439437866,
+      "learning_rate": 4.654059040590406e-06,
+      "loss": 1.6665,
+      "step": 406
+    },
+    {
+      "epoch": 0.5065339141257,
+      "grad_norm": 2.789813756942749,
+      "learning_rate": 4.651752767527676e-06,
+      "loss": 1.6064,
+      "step": 407
+    },
+    {
+      "epoch": 0.507778469197262,
+      "grad_norm": 3.3106348514556885,
+      "learning_rate": 4.649446494464945e-06,
+      "loss": 1.6542,
+      "step": 408
+    },
+    {
+      "epoch": 0.5090230242688238,
+      "grad_norm": 2.8805112838745117,
+      "learning_rate": 4.6471402214022145e-06,
+      "loss": 1.6058,
+      "step": 409
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "grad_norm": 3.30898118019104,
+      "learning_rate": 4.644833948339484e-06,
+      "loss": 1.5933,
+      "step": 410
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "eval_loss": 1.7787818908691406,
+      "eval_runtime": 44.0876,
+      "eval_samples_per_second": 22.682,
+      "eval_steps_per_second": 0.953,
+      "step": 410
+    },
+    {
+      "epoch": 0.5115121344119478,
+      "grad_norm": 3.3563625812530518,
+      "learning_rate": 4.642527675276753e-06,
+      "loss": 1.6751,
+      "step": 411
+    },
+    {
+      "epoch": 0.5127566894835096,
+      "grad_norm": 2.968120574951172,
+      "learning_rate": 4.640221402214023e-06,
+      "loss": 1.663,
+      "step": 412
+    },
+    {
+      "epoch": 0.5140012445550716,
+      "grad_norm": 3.310248374938965,
+      "learning_rate": 4.637915129151292e-06,
+      "loss": 1.658,
+      "step": 413
+    },
+    {
+      "epoch": 0.5152457996266335,
+      "grad_norm": 2.7011170387268066,
+      "learning_rate": 4.635608856088562e-06,
+      "loss": 1.6894,
+      "step": 414
+    },
+    {
+      "epoch": 0.5164903546981954,
+      "grad_norm": 2.804901361465454,
+      "learning_rate": 4.63330258302583e-06,
+      "loss": 1.602,
+      "step": 415
+    },
+    {
+      "epoch": 0.5177349097697573,
+      "grad_norm": 3.2412750720977783,
+      "learning_rate": 4.6309963099631e-06,
+      "loss": 1.6158,
+      "step": 416
+    },
+    {
+      "epoch": 0.5189794648413192,
+      "grad_norm": 2.699909210205078,
+      "learning_rate": 4.62869003690037e-06,
+      "loss": 1.5842,
+      "step": 417
+    },
+    {
+      "epoch": 0.5202240199128811,
+      "grad_norm": 3.065929412841797,
+      "learning_rate": 4.6263837638376384e-06,
+      "loss": 1.6434,
+      "step": 418
+    },
+    {
+      "epoch": 0.5214685749844431,
+      "grad_norm": 2.895305633544922,
+      "learning_rate": 4.624077490774908e-06,
+      "loss": 1.5969,
+      "step": 419
+    },
+    {
+      "epoch": 0.522713130056005,
+      "grad_norm": 3.1590664386749268,
+      "learning_rate": 4.621771217712177e-06,
+      "loss": 1.6211,
+      "step": 420
+    },
+    {
+      "epoch": 0.522713130056005,
+      "eval_loss": 1.7674189805984497,
+      "eval_runtime": 53.328,
+      "eval_samples_per_second": 18.752,
+      "eval_steps_per_second": 0.788,
+      "step": 420
+    },
+    {
+      "epoch": 0.5239576851275669,
+      "grad_norm": 2.7126312255859375,
+      "learning_rate": 4.619464944649447e-06,
+      "loss": 1.6246,
+      "step": 421
+    },
+    {
+      "epoch": 0.5252022401991289,
+      "grad_norm": 2.918508768081665,
+      "learning_rate": 4.617158671586716e-06,
+      "loss": 1.6937,
+      "step": 422
+    },
+    {
+      "epoch": 0.5264467952706907,
+      "grad_norm": 3.008610725402832,
+      "learning_rate": 4.6148523985239856e-06,
+      "loss": 1.6864,
+      "step": 423
+    },
+    {
+      "epoch": 0.5276913503422527,
+      "grad_norm": 2.6612043380737305,
+      "learning_rate": 4.612546125461255e-06,
+      "loss": 1.5785,
+      "step": 424
+    },
+    {
+      "epoch": 0.5289359054138145,
+      "grad_norm": 2.850679874420166,
+      "learning_rate": 4.610239852398524e-06,
+      "loss": 1.5899,
+      "step": 425
+    },
+    {
+      "epoch": 0.5301804604853765,
+      "grad_norm": 2.9205031394958496,
+      "learning_rate": 4.607933579335794e-06,
+      "loss": 1.5867,
+      "step": 426
+    },
+    {
+      "epoch": 0.5314250155569384,
+      "grad_norm": 2.929532527923584,
+      "learning_rate": 4.605627306273063e-06,
+      "loss": 1.6182,
+      "step": 427
+    },
+    {
+      "epoch": 0.5326695706285003,
+      "grad_norm": 2.874936103820801,
+      "learning_rate": 4.603321033210333e-06,
+      "loss": 1.5789,
+      "step": 428
+    },
+    {
+      "epoch": 0.5339141257000622,
+      "grad_norm": 2.8703525066375732,
+      "learning_rate": 4.601014760147602e-06,
+      "loss": 1.5859,
+      "step": 429
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "grad_norm": 2.7076902389526367,
+      "learning_rate": 4.5987084870848715e-06,
+      "loss": 1.644,
+      "step": 430
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "eval_loss": 1.7592095136642456,
+      "eval_runtime": 47.9438,
+      "eval_samples_per_second": 20.858,
+      "eval_steps_per_second": 0.876,
+      "step": 430
+    },
+    {
+      "epoch": 0.536403235843186,
+      "grad_norm": 2.839160680770874,
+      "learning_rate": 4.59640221402214e-06,
+      "loss": 1.6357,
+      "step": 431
+    },
+    {
+      "epoch": 0.537647790914748,
+      "grad_norm": 2.674025535583496,
+      "learning_rate": 4.5940959409594095e-06,
+      "loss": 1.676,
+      "step": 432
+    },
+    {
+      "epoch": 0.5388923459863099,
+      "grad_norm": 3.0619537830352783,
+      "learning_rate": 4.59178966789668e-06,
+      "loss": 1.5617,
+      "step": 433
+    },
+    {
+      "epoch": 0.5401369010578718,
+      "grad_norm": 3.058418035507202,
+      "learning_rate": 4.589483394833948e-06,
+      "loss": 1.5951,
+      "step": 434
+    },
+    {
+      "epoch": 0.5413814561294338,
+      "grad_norm": 3.0046396255493164,
+      "learning_rate": 4.587177121771218e-06,
+      "loss": 1.6473,
+      "step": 435
+    },
+    {
+      "epoch": 0.5426260112009956,
+      "grad_norm": 2.7736752033233643,
+      "learning_rate": 4.584870848708487e-06,
+      "loss": 1.6262,
+      "step": 436
+    },
+    {
+      "epoch": 0.5438705662725576,
+      "grad_norm": 2.7804994583129883,
+      "learning_rate": 4.5825645756457575e-06,
+      "loss": 1.6886,
+      "step": 437
+    },
+    {
+      "epoch": 0.5451151213441194,
+      "grad_norm": 3.0717954635620117,
+      "learning_rate": 4.580258302583026e-06,
+      "loss": 1.6287,
+      "step": 438
+    },
+    {
+      "epoch": 0.5463596764156814,
+      "grad_norm": 2.800243854522705,
+      "learning_rate": 4.5779520295202955e-06,
+      "loss": 1.6169,
+      "step": 439
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "grad_norm": 2.9581644535064697,
+      "learning_rate": 4.575645756457565e-06,
+      "loss": 1.6617,
+      "step": 440
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "eval_loss": 1.7642868757247925,
+      "eval_runtime": 51.2646,
+      "eval_samples_per_second": 19.507,
+      "eval_steps_per_second": 0.819,
+      "step": 440
+    },
+    {
+      "epoch": 0.5488487865588052,
+      "grad_norm": 3.0067460536956787,
+      "learning_rate": 4.573339483394834e-06,
+      "loss": 1.6482,
+      "step": 441
+    },
+    {
+      "epoch": 0.5500933416303672,
+      "grad_norm": 2.9075019359588623,
+      "learning_rate": 4.571033210332104e-06,
+      "loss": 1.6595,
+      "step": 442
+    },
+    {
+      "epoch": 0.551337896701929,
+      "grad_norm": 2.7788755893707275,
+      "learning_rate": 4.568726937269373e-06,
+      "loss": 1.5733,
+      "step": 443
+    },
+    {
+      "epoch": 0.552582451773491,
+      "grad_norm": 2.7775425910949707,
+      "learning_rate": 4.566420664206643e-06,
+      "loss": 1.6925,
+      "step": 444
+    },
+    {
+      "epoch": 0.5538270068450529,
+      "grad_norm": 2.6949567794799805,
+      "learning_rate": 4.564114391143912e-06,
+      "loss": 1.5738,
+      "step": 445
+    },
+    {
+      "epoch": 0.5550715619166148,
+      "grad_norm": 2.7093007564544678,
+      "learning_rate": 4.5618081180811814e-06,
+      "loss": 1.5848,
+      "step": 446
+    },
+    {
+      "epoch": 0.5563161169881767,
+      "grad_norm": 2.6920173168182373,
+      "learning_rate": 4.559501845018451e-06,
+      "loss": 1.6092,
+      "step": 447
+    },
+    {
+      "epoch": 0.5575606720597387,
+      "grad_norm": 2.733731985092163,
+      "learning_rate": 4.5571955719557194e-06,
+      "loss": 1.6056,
+      "step": 448
+    },
+    {
+      "epoch": 0.5588052271313005,
+      "grad_norm": 2.9156274795532227,
+      "learning_rate": 4.55488929889299e-06,
+      "loss": 1.6436,
+      "step": 449
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "grad_norm": 2.8802616596221924,
+      "learning_rate": 4.552583025830259e-06,
+      "loss": 1.6309,
+      "step": 450
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "eval_loss": 1.7486340999603271,
+      "eval_runtime": 47.2219,
+      "eval_samples_per_second": 21.177,
+      "eval_steps_per_second": 0.889,
+      "step": 450
+    },
+    {
+      "epoch": 0.5612943372744243,
+      "grad_norm": 2.729743242263794,
+      "learning_rate": 4.550276752767528e-06,
+      "loss": 1.7171,
+      "step": 451
+    },
+    {
+      "epoch": 0.5625388923459863,
+      "grad_norm": 2.800049066543579,
+      "learning_rate": 4.547970479704797e-06,
+      "loss": 1.6098,
+      "step": 452
+    },
+    {
+      "epoch": 0.5637834474175483,
+      "grad_norm": 2.9659311771392822,
+      "learning_rate": 4.5456642066420666e-06,
+      "loss": 1.6646,
+      "step": 453
+    },
+    {
+      "epoch": 0.5650280024891101,
+      "grad_norm": 2.5417754650115967,
+      "learning_rate": 4.543357933579336e-06,
+      "loss": 1.5847,
+      "step": 454
+    },
+    {
+      "epoch": 0.5662725575606721,
+      "grad_norm": 2.8471338748931885,
+      "learning_rate": 4.541051660516605e-06,
+      "loss": 1.6235,
+      "step": 455
+    },
+    {
+      "epoch": 0.5675171126322339,
+      "grad_norm": 2.7039637565612793,
+      "learning_rate": 4.538745387453875e-06,
+      "loss": 1.5613,
+      "step": 456
+    },
+    {
+      "epoch": 0.5687616677037959,
+      "grad_norm": 3.0383341312408447,
+      "learning_rate": 4.536439114391144e-06,
+      "loss": 1.6485,
+      "step": 457
+    },
+    {
+      "epoch": 0.5700062227753578,
+      "grad_norm": 2.528388261795044,
+      "learning_rate": 4.534132841328414e-06,
+      "loss": 1.6117,
+      "step": 458
+    },
+    {
+      "epoch": 0.5712507778469197,
+      "grad_norm": 2.7623119354248047,
+      "learning_rate": 4.531826568265683e-06,
+      "loss": 1.6017,
+      "step": 459
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "grad_norm": 2.9213945865631104,
+      "learning_rate": 4.5295202952029525e-06,
+      "loss": 1.643,
+      "step": 460
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "eval_loss": 1.747575044631958,
+      "eval_runtime": 45.2844,
+      "eval_samples_per_second": 22.083,
+      "eval_steps_per_second": 0.927,
+      "step": 460
+    },
+    {
+      "epoch": 0.5737398879900436,
+      "grad_norm": 2.664275884628296,
+      "learning_rate": 4.527214022140222e-06,
+      "loss": 1.6154,
+      "step": 461
+    },
+    {
+      "epoch": 0.5749844430616055,
+      "grad_norm": 2.5891788005828857,
+      "learning_rate": 4.524907749077491e-06,
+      "loss": 1.5975,
+      "step": 462
+    },
+    {
+      "epoch": 0.5762289981331674,
+      "grad_norm": 2.8126487731933594,
+      "learning_rate": 4.522601476014761e-06,
+      "loss": 1.6306,
+      "step": 463
+    },
+    {
+      "epoch": 0.5774735532047293,
+      "grad_norm": 2.799391031265259,
+      "learning_rate": 4.520295202952029e-06,
+      "loss": 1.6139,
+      "step": 464
+    },
+    {
+      "epoch": 0.5787181082762912,
+      "grad_norm": 2.7323718070983887,
+      "learning_rate": 4.5179889298893e-06,
+      "loss": 1.632,
+      "step": 465
+    },
+    {
+      "epoch": 0.5799626633478532,
+      "grad_norm": 2.601386785507202,
+      "learning_rate": 4.515682656826569e-06,
+      "loss": 1.6139,
+      "step": 466
+    },
+    {
+      "epoch": 0.581207218419415,
+      "grad_norm": 2.562162399291992,
+      "learning_rate": 4.513376383763838e-06,
+      "loss": 1.5168,
+      "step": 467
+    },
+    {
+      "epoch": 0.582451773490977,
+      "grad_norm": 2.6912002563476562,
+      "learning_rate": 4.511070110701107e-06,
+      "loss": 1.6249,
+      "step": 468
+    },
+    {
+      "epoch": 0.583696328562539,
+      "grad_norm": 2.6971495151519775,
+      "learning_rate": 4.5087638376383765e-06,
+      "loss": 1.616,
+      "step": 469
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "grad_norm": 2.6898910999298096,
+      "learning_rate": 4.506457564575646e-06,
+      "loss": 1.614,
+      "step": 470
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "eval_loss": 1.7460769414901733,
+      "eval_runtime": 45.3055,
+      "eval_samples_per_second": 22.072,
+      "eval_steps_per_second": 0.927,
+      "step": 470
+    },
+    {
+      "epoch": 0.5861854387056628,
+      "grad_norm": 2.698180675506592,
+      "learning_rate": 4.504151291512915e-06,
+      "loss": 1.5993,
+      "step": 471
+    },
+    {
+      "epoch": 0.5874299937772246,
+      "grad_norm": 2.7125210762023926,
+      "learning_rate": 4.501845018450185e-06,
+      "loss": 1.6113,
+      "step": 472
+    },
+    {
+      "epoch": 0.5886745488487866,
+      "grad_norm": 2.754445791244507,
+      "learning_rate": 4.499538745387454e-06,
+      "loss": 1.6719,
+      "step": 473
+    },
+    {
+      "epoch": 0.5899191039203485,
+      "grad_norm": 2.6979362964630127,
+      "learning_rate": 4.497232472324724e-06,
+      "loss": 1.608,
+      "step": 474
+    },
+    {
+      "epoch": 0.5911636589919104,
+      "grad_norm": 2.7694528102874756,
+      "learning_rate": 4.494926199261993e-06,
+      "loss": 1.5695,
+      "step": 475
+    },
+    {
+      "epoch": 0.5924082140634723,
+      "grad_norm": 2.653353214263916,
+      "learning_rate": 4.4926199261992624e-06,
+      "loss": 1.5523,
+      "step": 476
+    },
+    {
+      "epoch": 0.5936527691350342,
+      "grad_norm": 2.6632070541381836,
+      "learning_rate": 4.490313653136532e-06,
+      "loss": 1.5675,
+      "step": 477
+    },
+    {
+      "epoch": 0.5948973242065961,
+      "grad_norm": 3.038543939590454,
+      "learning_rate": 4.488007380073801e-06,
+      "loss": 1.5692,
+      "step": 478
+    },
+    {
+      "epoch": 0.5961418792781581,
+      "grad_norm": 2.8123953342437744,
+      "learning_rate": 4.485701107011071e-06,
+      "loss": 1.6223,
+      "step": 479
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "grad_norm": 2.5845773220062256,
+      "learning_rate": 4.483394833948339e-06,
+      "loss": 1.5773,
+      "step": 480
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "eval_loss": 1.7419319152832031,
+      "eval_runtime": 44.6842,
+      "eval_samples_per_second": 22.379,
+      "eval_steps_per_second": 0.94,
+      "step": 480
+    },
+    {
+      "epoch": 0.5986309894212819,
+      "grad_norm": 3.069936752319336,
+      "learning_rate": 4.4810885608856096e-06,
+      "loss": 1.6323,
+      "step": 481
+    },
+    {
+      "epoch": 0.5998755444928439,
+      "grad_norm": 2.9085781574249268,
+      "learning_rate": 4.478782287822879e-06,
+      "loss": 1.5845,
+      "step": 482
+    },
+    {
+      "epoch": 0.6011200995644057,
+      "grad_norm": 2.8568010330200195,
+      "learning_rate": 4.4764760147601476e-06,
+      "loss": 1.5898,
+      "step": 483
+    },
+    {
+      "epoch": 0.6023646546359677,
+      "grad_norm": 3.089081048965454,
+      "learning_rate": 4.474169741697417e-06,
+      "loss": 1.6622,
+      "step": 484
+    },
+    {
+      "epoch": 0.6036092097075295,
+      "grad_norm": 2.8515470027923584,
+      "learning_rate": 4.471863468634686e-06,
+      "loss": 1.5859,
+      "step": 485
+    },
+    {
+      "epoch": 0.6048537647790915,
+      "grad_norm": 2.647765874862671,
+      "learning_rate": 4.469557195571957e-06,
+      "loss": 1.5926,
+      "step": 486
+    },
+    {
+      "epoch": 0.6060983198506534,
+      "grad_norm": 2.61676025390625,
+      "learning_rate": 4.467250922509225e-06,
+      "loss": 1.5684,
+      "step": 487
+    },
+    {
+      "epoch": 0.6073428749222153,
+      "grad_norm": 2.7127039432525635,
+      "learning_rate": 4.464944649446495e-06,
+      "loss": 1.5702,
+      "step": 488
+    },
+    {
+      "epoch": 0.6085874299937772,
+      "grad_norm": 2.9932055473327637,
+      "learning_rate": 4.462638376383764e-06,
+      "loss": 1.5976,
+      "step": 489
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "grad_norm": 2.6508774757385254,
+      "learning_rate": 4.4603321033210335e-06,
+      "loss": 1.586,
+      "step": 490
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "eval_loss": 1.7357326745986938,
+      "eval_runtime": 45.393,
+      "eval_samples_per_second": 22.03,
+      "eval_steps_per_second": 0.925,
+      "step": 490
+    },
+    {
+      "epoch": 0.611076540136901,
+      "grad_norm": 2.7626044750213623,
+      "learning_rate": 4.458025830258303e-06,
+      "loss": 1.5773,
+      "step": 491
+    },
+    {
+      "epoch": 0.612321095208463,
+      "grad_norm": 2.595003604888916,
+      "learning_rate": 4.455719557195572e-06,
+      "loss": 1.601,
+      "step": 492
+    },
+    {
+      "epoch": 0.6135656502800249,
+      "grad_norm": 2.595767021179199,
+      "learning_rate": 4.453413284132842e-06,
+      "loss": 1.6287,
+      "step": 493
+    },
+    {
+      "epoch": 0.6148102053515868,
+      "grad_norm": 2.755845308303833,
+      "learning_rate": 4.451107011070111e-06,
+      "loss": 1.593,
+      "step": 494
+    },
+    {
+      "epoch": 0.6160547604231488,
+      "grad_norm": 2.7302653789520264,
+      "learning_rate": 4.448800738007381e-06,
+      "loss": 1.5768,
+      "step": 495
+    },
+    {
+      "epoch": 0.6172993154947106,
+      "grad_norm": 2.6493024826049805,
+      "learning_rate": 4.446494464944649e-06,
+      "loss": 1.6059,
+      "step": 496
+    },
+    {
+      "epoch": 0.6185438705662726,
+      "grad_norm": 2.8633735179901123,
+      "learning_rate": 4.4441881918819195e-06,
+      "loss": 1.598,
+      "step": 497
+    },
+    {
+      "epoch": 0.6197884256378344,
+      "grad_norm": 2.65639066696167,
+      "learning_rate": 4.441881918819189e-06,
+      "loss": 1.6036,
+      "step": 498
+    },
+    {
+      "epoch": 0.6210329807093964,
+      "grad_norm": 2.6867759227752686,
+      "learning_rate": 4.439575645756458e-06,
+      "loss": 1.5676,
+      "step": 499
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "grad_norm": 2.7986443042755127,
+      "learning_rate": 4.437269372693727e-06,
+      "loss": 1.524,
+      "step": 500
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "eval_loss": 1.7449125051498413,
+      "eval_runtime": 43.9329,
+      "eval_samples_per_second": 22.762,
+      "eval_steps_per_second": 0.956,
+      "step": 500
+    },
+    {
+      "epoch": 0.6235220908525202,
+      "grad_norm": 2.665905714035034,
+      "learning_rate": 4.434963099630996e-06,
+      "loss": 1.5635,
+      "step": 501
+    },
+    {
+      "epoch": 0.6247666459240822,
+      "grad_norm": 2.902435541152954,
+      "learning_rate": 4.432656826568267e-06,
+      "loss": 1.6377,
+      "step": 502
+    },
+    {
+      "epoch": 0.6260112009956441,
+      "grad_norm": 2.6872262954711914,
+      "learning_rate": 4.430350553505535e-06,
+      "loss": 1.5553,
+      "step": 503
+    },
+    {
+      "epoch": 0.627255756067206,
+      "grad_norm": 2.676621913909912,
+      "learning_rate": 4.428044280442805e-06,
+      "loss": 1.6101,
+      "step": 504
+    },
+    {
+      "epoch": 0.6285003111387679,
+      "grad_norm": 2.756347179412842,
+      "learning_rate": 4.425738007380074e-06,
+      "loss": 1.5884,
+      "step": 505
+    },
+    {
+      "epoch": 0.6297448662103298,
+      "grad_norm": 2.856882333755493,
+      "learning_rate": 4.4234317343173434e-06,
+      "loss": 1.5635,
+      "step": 506
+    },
+    {
+      "epoch": 0.6309894212818917,
+      "grad_norm": 2.7708330154418945,
+      "learning_rate": 4.421125461254613e-06,
+      "loss": 1.6028,
+      "step": 507
+    },
+    {
+      "epoch": 0.6322339763534537,
+      "grad_norm": 2.8167600631713867,
+      "learning_rate": 4.418819188191882e-06,
+      "loss": 1.6026,
+      "step": 508
+    },
+    {
+      "epoch": 0.6334785314250155,
+      "grad_norm": 2.54194974899292,
+      "learning_rate": 4.416512915129152e-06,
+      "loss": 1.6001,
+      "step": 509
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "grad_norm": 2.683037519454956,
+      "learning_rate": 4.414206642066421e-06,
+      "loss": 1.6084,
+      "step": 510
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "eval_loss": 1.7376186847686768,
+      "eval_runtime": 42.3026,
+      "eval_samples_per_second": 23.639,
+      "eval_steps_per_second": 0.993,
+      "step": 510
+    },
+    {
+      "epoch": 0.6359676415681393,
+      "grad_norm": 2.82627010345459,
+      "learning_rate": 4.4119003690036905e-06,
+      "loss": 1.6258,
+      "step": 511
+    },
+    {
+      "epoch": 0.6372121966397013,
+      "grad_norm": 2.6139848232269287,
+      "learning_rate": 4.40959409594096e-06,
+      "loss": 1.6029,
+      "step": 512
+    },
+    {
+      "epoch": 0.6384567517112633,
+      "grad_norm": 2.646712303161621,
+      "learning_rate": 4.407287822878229e-06,
+      "loss": 1.5602,
+      "step": 513
+    },
+    {
+      "epoch": 0.6397013067828251,
+      "grad_norm": 2.910935163497925,
+      "learning_rate": 4.404981549815499e-06,
+      "loss": 1.5757,
+      "step": 514
+    },
+    {
+      "epoch": 0.6409458618543871,
+      "grad_norm": 2.6001148223876953,
+      "learning_rate": 4.402675276752768e-06,
+      "loss": 1.5857,
+      "step": 515
+    },
+    {
+      "epoch": 0.642190416925949,
+      "grad_norm": 2.8525235652923584,
+      "learning_rate": 4.400369003690037e-06,
+      "loss": 1.6092,
+      "step": 516
+    },
+    {
+      "epoch": 0.6434349719975109,
+      "grad_norm": 2.5335488319396973,
+      "learning_rate": 4.398062730627306e-06,
+      "loss": 1.5849,
+      "step": 517
+    },
+    {
+      "epoch": 0.6446795270690728,
+      "grad_norm": 2.5787103176116943,
+      "learning_rate": 4.3957564575645765e-06,
+      "loss": 1.634,
+      "step": 518
+    },
+    {
+      "epoch": 0.6459240821406347,
+      "grad_norm": 2.6188197135925293,
+      "learning_rate": 4.393450184501845e-06,
+      "loss": 1.5477,
+      "step": 519
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "grad_norm": 2.6548666954040527,
+      "learning_rate": 4.3911439114391145e-06,
+      "loss": 1.6133,
+      "step": 520
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "eval_loss": 1.7393039464950562,
+      "eval_runtime": 45.9234,
+      "eval_samples_per_second": 21.775,
+      "eval_steps_per_second": 0.915,
+      "step": 520
+    },
+    {
+      "epoch": 0.6484131922837586,
+      "grad_norm": 2.907928228378296,
+      "learning_rate": 4.388837638376384e-06,
+      "loss": 1.587,
+      "step": 521
+    },
+    {
+      "epoch": 0.6496577473553204,
+      "grad_norm": 2.7899692058563232,
+      "learning_rate": 4.386531365313653e-06,
+      "loss": 1.6126,
+      "step": 522
+    },
+    {
+      "epoch": 0.6509023024268824,
+      "grad_norm": 2.680147886276245,
+      "learning_rate": 4.384225092250923e-06,
+      "loss": 1.565,
+      "step": 523
+    },
+    {
+      "epoch": 0.6521468574984443,
+      "grad_norm": 2.6590754985809326,
+      "learning_rate": 4.381918819188192e-06,
+      "loss": 1.6127,
+      "step": 524
+    },
+    {
+      "epoch": 0.6533914125700062,
+      "grad_norm": 2.6638906002044678,
+      "learning_rate": 4.379612546125462e-06,
+      "loss": 1.5615,
+      "step": 525
+    },
+    {
+      "epoch": 0.6546359676415682,
+      "grad_norm": 2.6518194675445557,
+      "learning_rate": 4.377306273062731e-06,
+      "loss": 1.6202,
+      "step": 526
+    },
+    {
+      "epoch": 0.65588052271313,
+      "grad_norm": 2.5323619842529297,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 1.5305,
+      "step": 527
+    },
+    {
+      "epoch": 0.657125077784692,
+      "grad_norm": 2.6871442794799805,
+      "learning_rate": 4.37269372693727e-06,
+      "loss": 1.6043,
+      "step": 528
+    },
+    {
+      "epoch": 0.6583696328562539,
+      "grad_norm": 2.760746717453003,
+      "learning_rate": 4.370387453874539e-06,
+      "loss": 1.6059,
+      "step": 529
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "grad_norm": 2.780482769012451,
+      "learning_rate": 4.368081180811809e-06,
+      "loss": 1.6082,
+      "step": 530
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "eval_loss": 1.7219713926315308,
+      "eval_runtime": 51.8478,
+      "eval_samples_per_second": 19.287,
+      "eval_steps_per_second": 0.81,
+      "step": 530
+    },
+    {
+      "epoch": 0.6608587429993777,
+      "grad_norm": 2.605890989303589,
+      "learning_rate": 4.365774907749078e-06,
+      "loss": 1.5554,
+      "step": 531
+    },
+    {
+      "epoch": 0.6621032980709396,
+      "grad_norm": 2.731555461883545,
+      "learning_rate": 4.363468634686347e-06,
+      "loss": 1.606,
+      "step": 532
+    },
+    {
+      "epoch": 0.6633478531425016,
+      "grad_norm": 2.8356943130493164,
+      "learning_rate": 4.361162361623616e-06,
+      "loss": 1.5601,
+      "step": 533
+    },
+    {
+      "epoch": 0.6645924082140635,
+      "grad_norm": 2.7196593284606934,
+      "learning_rate": 4.3588560885608864e-06,
+      "loss": 1.5722,
+      "step": 534
+    },
+    {
+      "epoch": 0.6658369632856254,
+      "grad_norm": 2.621371269226074,
+      "learning_rate": 4.356549815498156e-06,
+      "loss": 1.6194,
+      "step": 535
+    },
+    {
+      "epoch": 0.6670815183571873,
+      "grad_norm": 2.653916120529175,
+      "learning_rate": 4.354243542435424e-06,
+      "loss": 1.5592,
+      "step": 536
+    },
+    {
+      "epoch": 0.6683260734287492,
+      "grad_norm": 2.88431453704834,
+      "learning_rate": 4.351937269372694e-06,
+      "loss": 1.6296,
+      "step": 537
+    },
+    {
+      "epoch": 0.6695706285003111,
+      "grad_norm": 2.667130470275879,
+      "learning_rate": 4.349630996309963e-06,
+      "loss": 1.5624,
+      "step": 538
+    },
+    {
+      "epoch": 0.6708151835718731,
+      "grad_norm": 2.6453566551208496,
+      "learning_rate": 4.347324723247233e-06,
+      "loss": 1.6025,
+      "step": 539
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "grad_norm": 3.032271146774292,
+      "learning_rate": 4.345018450184502e-06,
+      "loss": 1.6771,
+      "step": 540
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "eval_loss": 1.721895456314087,
+      "eval_runtime": 50.62,
+      "eval_samples_per_second": 19.755,
+      "eval_steps_per_second": 0.83,
+      "step": 540
+    },
+    {
+      "epoch": 0.6733042937149969,
+      "grad_norm": 2.8476362228393555,
+      "learning_rate": 4.3427121771217715e-06,
+      "loss": 1.5663,
+      "step": 541
+    },
+    {
+      "epoch": 0.6745488487865589,
+      "grad_norm": 2.615602970123291,
+      "learning_rate": 4.340405904059041e-06,
+      "loss": 1.5815,
+      "step": 542
+    },
+    {
+      "epoch": 0.6757934038581207,
+      "grad_norm": 3.221571683883667,
+      "learning_rate": 4.33809963099631e-06,
+      "loss": 1.5955,
+      "step": 543
+    },
+    {
+      "epoch": 0.6770379589296827,
+      "grad_norm": 2.602994203567505,
+      "learning_rate": 4.33579335793358e-06,
+      "loss": 1.5667,
+      "step": 544
+    },
+    {
+      "epoch": 0.6782825140012445,
+      "grad_norm": 2.9380433559417725,
+      "learning_rate": 4.333487084870848e-06,
+      "loss": 1.5651,
+      "step": 545
+    },
+    {
+      "epoch": 0.6795270690728065,
+      "grad_norm": 3.0689425468444824,
+      "learning_rate": 4.331180811808119e-06,
+      "loss": 1.6003,
+      "step": 546
+    },
+    {
+      "epoch": 0.6807716241443684,
+      "grad_norm": 2.770317316055298,
+      "learning_rate": 4.328874538745388e-06,
+      "loss": 1.5576,
+      "step": 547
+    },
+    {
+      "epoch": 0.6820161792159303,
+      "grad_norm": 2.8631815910339355,
+      "learning_rate": 4.3265682656826575e-06,
+      "loss": 1.5308,
+      "step": 548
+    },
+    {
+      "epoch": 0.6832607342874922,
+      "grad_norm": 2.785576820373535,
+      "learning_rate": 4.324261992619926e-06,
+      "loss": 1.6113,
+      "step": 549
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "grad_norm": 2.723919630050659,
+      "learning_rate": 4.321955719557196e-06,
+      "loss": 1.5606,
+      "step": 550
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "eval_loss": 1.721803069114685,
+      "eval_runtime": 51.6036,
+      "eval_samples_per_second": 19.378,
+      "eval_steps_per_second": 0.814,
+      "step": 550
+    },
+    {
+      "epoch": 0.685749844430616,
+      "grad_norm": 3.081198215484619,
+      "learning_rate": 4.319649446494466e-06,
+      "loss": 1.5643,
+      "step": 551
+    },
+    {
+      "epoch": 0.686994399502178,
+      "grad_norm": 2.757392168045044,
+      "learning_rate": 4.317343173431734e-06,
+      "loss": 1.5909,
+      "step": 552
+    },
+    {
+      "epoch": 0.6882389545737398,
+      "grad_norm": 3.038753032684326,
+      "learning_rate": 4.315036900369004e-06,
+      "loss": 1.6023,
+      "step": 553
+    },
+    {
+      "epoch": 0.6894835096453018,
+      "grad_norm": 2.8246452808380127,
+      "learning_rate": 4.312730627306273e-06,
+      "loss": 1.5548,
+      "step": 554
+    },
+    {
+      "epoch": 0.6907280647168638,
+      "grad_norm": 2.589320182800293,
+      "learning_rate": 4.310424354243543e-06,
+      "loss": 1.5424,
+      "step": 555
+    },
+    {
+      "epoch": 0.6919726197884256,
+      "grad_norm": 2.6413373947143555,
+      "learning_rate": 4.308118081180812e-06,
+      "loss": 1.6243,
+      "step": 556
+    },
+    {
+      "epoch": 0.6932171748599876,
+      "grad_norm": 2.764784336090088,
+      "learning_rate": 4.3058118081180815e-06,
+      "loss": 1.589,
+      "step": 557
+    },
+    {
+      "epoch": 0.6944617299315494,
+      "grad_norm": 2.806821823120117,
+      "learning_rate": 4.303505535055351e-06,
+      "loss": 1.621,
+      "step": 558
+    },
+    {
+      "epoch": 0.6957062850031114,
+      "grad_norm": 2.718017578125,
+      "learning_rate": 4.30119926199262e-06,
+      "loss": 1.5773,
+      "step": 559
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "grad_norm": 2.7620160579681396,
+      "learning_rate": 4.29889298892989e-06,
+      "loss": 1.5868,
+      "step": 560
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "eval_loss": 1.7152249813079834,
+      "eval_runtime": 52.0463,
+      "eval_samples_per_second": 19.214,
+      "eval_steps_per_second": 0.807,
+      "step": 560
+    },
+    {
+      "epoch": 0.6981953951462352,
+      "grad_norm": 2.66686749458313,
+      "learning_rate": 4.296586715867159e-06,
+      "loss": 1.534,
+      "step": 561
+    },
+    {
+      "epoch": 0.6994399502177971,
+      "grad_norm": 2.812577247619629,
+      "learning_rate": 4.2942804428044286e-06,
+      "loss": 1.5501,
+      "step": 562
+    },
+    {
+      "epoch": 0.7006845052893591,
+      "grad_norm": 2.578508138656616,
+      "learning_rate": 4.291974169741698e-06,
+      "loss": 1.6286,
+      "step": 563
+    },
+    {
+      "epoch": 0.701929060360921,
+      "grad_norm": 2.524249315261841,
+      "learning_rate": 4.289667896678967e-06,
+      "loss": 1.5719,
+      "step": 564
+    },
+    {
+      "epoch": 0.7031736154324829,
+      "grad_norm": 2.827235460281372,
+      "learning_rate": 4.287361623616236e-06,
+      "loss": 1.5333,
+      "step": 565
+    },
+    {
+      "epoch": 0.7044181705040448,
+      "grad_norm": 2.6359963417053223,
+      "learning_rate": 4.285055350553506e-06,
+      "loss": 1.5849,
+      "step": 566
+    },
+    {
+      "epoch": 0.7056627255756067,
+      "grad_norm": 2.930530071258545,
+      "learning_rate": 4.282749077490776e-06,
+      "loss": 1.5672,
+      "step": 567
+    },
+    {
+      "epoch": 0.7069072806471687,
+      "grad_norm": 2.750102996826172,
+      "learning_rate": 4.280442804428044e-06,
+      "loss": 1.5696,
+      "step": 568
+    },
+    {
+      "epoch": 0.7081518357187305,
+      "grad_norm": 2.869690418243408,
+      "learning_rate": 4.278136531365314e-06,
+      "loss": 1.5689,
+      "step": 569
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "grad_norm": 2.954852819442749,
+      "learning_rate": 4.275830258302583e-06,
+      "loss": 1.5931,
+      "step": 570
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "eval_loss": 1.7143300771713257,
+      "eval_runtime": 52.7293,
+      "eval_samples_per_second": 18.965,
+      "eval_steps_per_second": 0.797,
+      "step": 570
+    },
+    {
+      "epoch": 0.7106409458618543,
+      "grad_norm": 2.702223539352417,
+      "learning_rate": 4.273523985239853e-06,
+      "loss": 1.5635,
+      "step": 571
+    },
+    {
+      "epoch": 0.7118855009334163,
+      "grad_norm": 2.689995050430298,
+      "learning_rate": 4.271217712177122e-06,
+      "loss": 1.5545,
+      "step": 572
+    },
+    {
+      "epoch": 0.7131300560049783,
+      "grad_norm": 2.68979811668396,
+      "learning_rate": 4.268911439114391e-06,
+      "loss": 1.5404,
+      "step": 573
+    },
+    {
+      "epoch": 0.7143746110765401,
+      "grad_norm": 2.7477986812591553,
+      "learning_rate": 4.266605166051661e-06,
+      "loss": 1.5719,
+      "step": 574
+    },
+    {
+      "epoch": 0.7156191661481021,
+      "grad_norm": 2.975778341293335,
+      "learning_rate": 4.26429889298893e-06,
+      "loss": 1.5587,
+      "step": 575
+    },
+    {
+      "epoch": 0.716863721219664,
+      "grad_norm": 2.658170700073242,
+      "learning_rate": 4.2619926199262e-06,
+      "loss": 1.5746,
+      "step": 576
+    },
+    {
+      "epoch": 0.7181082762912259,
+      "grad_norm": 2.8825011253356934,
+      "learning_rate": 4.259686346863469e-06,
+      "loss": 1.5246,
+      "step": 577
+    },
+    {
+      "epoch": 0.7193528313627878,
+      "grad_norm": 2.845280170440674,
+      "learning_rate": 4.2573800738007385e-06,
+      "loss": 1.5703,
+      "step": 578
+    },
+    {
+      "epoch": 0.7205973864343497,
+      "grad_norm": 2.660616636276245,
+      "learning_rate": 4.255073800738008e-06,
+      "loss": 1.5373,
+      "step": 579
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "grad_norm": 2.749447822570801,
+      "learning_rate": 4.252767527675277e-06,
+      "loss": 1.59,
+      "step": 580
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "eval_loss": 1.7149444818496704,
+      "eval_runtime": 52.2099,
+      "eval_samples_per_second": 19.153,
+      "eval_steps_per_second": 0.804,
+      "step": 580
+    },
+    {
+      "epoch": 0.7230864965774736,
+      "grad_norm": 2.813328742980957,
+      "learning_rate": 4.250461254612546e-06,
+      "loss": 1.5694,
+      "step": 581
+    },
+    {
+      "epoch": 0.7243310516490354,
+      "grad_norm": 2.823866844177246,
+      "learning_rate": 4.248154981549816e-06,
+      "loss": 1.5143,
+      "step": 582
+    },
+    {
+      "epoch": 0.7255756067205974,
+      "grad_norm": 2.71337890625,
+      "learning_rate": 4.245848708487086e-06,
+      "loss": 1.5915,
+      "step": 583
+    },
+    {
+      "epoch": 0.7268201617921594,
+      "grad_norm": 2.718085765838623,
+      "learning_rate": 4.243542435424355e-06,
+      "loss": 1.604,
+      "step": 584
+    },
+    {
+      "epoch": 0.7280647168637212,
+      "grad_norm": 2.5741796493530273,
+      "learning_rate": 4.241236162361624e-06,
+      "loss": 1.5398,
+      "step": 585
+    },
+    {
+      "epoch": 0.7293092719352832,
+      "grad_norm": 2.727114200592041,
+      "learning_rate": 4.238929889298893e-06,
+      "loss": 1.5886,
+      "step": 586
+    },
+    {
+      "epoch": 0.730553827006845,
+      "grad_norm": 2.6703338623046875,
+      "learning_rate": 4.236623616236163e-06,
+      "loss": 1.5959,
+      "step": 587
+    },
+    {
+      "epoch": 0.731798382078407,
+      "grad_norm": 2.750326633453369,
+      "learning_rate": 4.234317343173432e-06,
+      "loss": 1.5916,
+      "step": 588
+    },
+    {
+      "epoch": 0.7330429371499689,
+      "grad_norm": 2.6680715084075928,
+      "learning_rate": 4.232011070110701e-06,
+      "loss": 1.562,
+      "step": 589
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "grad_norm": 2.6603832244873047,
+      "learning_rate": 4.229704797047971e-06,
+      "loss": 1.6121,
+      "step": 590
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "eval_loss": 1.7144734859466553,
+      "eval_runtime": 50.5317,
+      "eval_samples_per_second": 19.79,
+      "eval_steps_per_second": 0.831,
+      "step": 590
+    },
+    {
+      "epoch": 0.7355320472930927,
+      "grad_norm": 2.7868523597717285,
+      "learning_rate": 4.22739852398524e-06,
+      "loss": 1.5488,
+      "step": 591
+    },
+    {
+      "epoch": 0.7367766023646546,
+      "grad_norm": 2.755270004272461,
+      "learning_rate": 4.2250922509225096e-06,
+      "loss": 1.5801,
+      "step": 592
+    },
+    {
+      "epoch": 0.7380211574362165,
+      "grad_norm": 2.7614126205444336,
+      "learning_rate": 4.222785977859779e-06,
+      "loss": 1.5507,
+      "step": 593
+    },
+    {
+      "epoch": 0.7392657125077785,
+      "grad_norm": 2.8674862384796143,
+      "learning_rate": 4.220479704797048e-06,
+      "loss": 1.5807,
+      "step": 594
+    },
+    {
+      "epoch": 0.7405102675793404,
+      "grad_norm": 2.7366859912872314,
+      "learning_rate": 4.218173431734318e-06,
+      "loss": 1.6594,
+      "step": 595
+    },
+    {
+      "epoch": 0.7417548226509023,
+      "grad_norm": 2.6231045722961426,
+      "learning_rate": 4.215867158671587e-06,
+      "loss": 1.5609,
+      "step": 596
+    },
+    {
+      "epoch": 0.7429993777224643,
+      "grad_norm": 2.7694146633148193,
+      "learning_rate": 4.213560885608857e-06,
+      "loss": 1.5889,
+      "step": 597
+    },
+    {
+      "epoch": 0.7442439327940261,
+      "grad_norm": 2.6275386810302734,
+      "learning_rate": 4.211254612546125e-06,
+      "loss": 1.575,
+      "step": 598
+    },
+    {
+      "epoch": 0.7454884878655881,
+      "grad_norm": 2.9413866996765137,
+      "learning_rate": 4.2089483394833955e-06,
+      "loss": 1.589,
+      "step": 599
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "grad_norm": 2.768606185913086,
+      "learning_rate": 4.206642066420665e-06,
+      "loss": 1.5738,
+      "step": 600
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "eval_loss": 1.7148027420043945,
+      "eval_runtime": 54.7031,
+      "eval_samples_per_second": 18.28,
+      "eval_steps_per_second": 0.768,
+      "step": 600
+    },
+    {
+      "epoch": 0.7479775980087119,
+      "grad_norm": 2.7917706966400146,
+      "learning_rate": 4.2043357933579335e-06,
+      "loss": 1.499,
+      "step": 601
+    },
+    {
+      "epoch": 0.7492221530802738,
+      "grad_norm": 2.5982208251953125,
+      "learning_rate": 4.202029520295203e-06,
+      "loss": 1.5113,
+      "step": 602
+    },
+    {
+      "epoch": 0.7504667081518357,
+      "grad_norm": 2.692032814025879,
+      "learning_rate": 4.199723247232473e-06,
+      "loss": 1.4721,
+      "step": 603
+    },
+    {
+      "epoch": 0.7517112632233977,
+      "grad_norm": 2.975860834121704,
+      "learning_rate": 4.197416974169742e-06,
+      "loss": 1.5147,
+      "step": 604
+    },
+    {
+      "epoch": 0.7529558182949595,
+      "grad_norm": 2.758610248565674,
+      "learning_rate": 4.195110701107011e-06,
+      "loss": 1.5674,
+      "step": 605
+    },
+    {
+      "epoch": 0.7542003733665215,
+      "grad_norm": 2.7620184421539307,
+      "learning_rate": 4.192804428044281e-06,
+      "loss": 1.5669,
+      "step": 606
+    },
+    {
+      "epoch": 0.7554449284380834,
+      "grad_norm": 3.043940305709839,
+      "learning_rate": 4.19049815498155e-06,
+      "loss": 1.587,
+      "step": 607
+    },
+    {
+      "epoch": 0.7566894835096453,
+      "grad_norm": 2.7482151985168457,
+      "learning_rate": 4.1881918819188195e-06,
+      "loss": 1.5451,
+      "step": 608
+    },
+    {
+      "epoch": 0.7579340385812072,
+      "grad_norm": 2.7259294986724854,
+      "learning_rate": 4.185885608856089e-06,
+      "loss": 1.5295,
+      "step": 609
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "grad_norm": 2.6340832710266113,
+      "learning_rate": 4.183579335793358e-06,
+      "loss": 1.5335,
+      "step": 610
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "eval_loss": 1.7038393020629883,
+      "eval_runtime": 44.0792,
+      "eval_samples_per_second": 22.686,
+      "eval_steps_per_second": 0.953,
+      "step": 610
+    },
+    {
+      "epoch": 0.760423148724331,
+      "grad_norm": 2.6144909858703613,
+      "learning_rate": 4.181273062730628e-06,
+      "loss": 1.5907,
+      "step": 611
+    },
+    {
+      "epoch": 0.761667703795893,
+      "grad_norm": 2.7754175662994385,
+      "learning_rate": 4.178966789667897e-06,
+      "loss": 1.5773,
+      "step": 612
+    },
+    {
+      "epoch": 0.7629122588674548,
+      "grad_norm": 2.6313252449035645,
+      "learning_rate": 4.176660516605167e-06,
+      "loss": 1.5675,
+      "step": 613
+    },
+    {
+      "epoch": 0.7641568139390168,
+      "grad_norm": 2.549074649810791,
+      "learning_rate": 4.174354243542435e-06,
+      "loss": 1.595,
+      "step": 614
+    },
+    {
+      "epoch": 0.7654013690105788,
+      "grad_norm": 2.611804246902466,
+      "learning_rate": 4.1720479704797054e-06,
+      "loss": 1.5416,
+      "step": 615
+    },
+    {
+      "epoch": 0.7666459240821406,
+      "grad_norm": 2.6322927474975586,
+      "learning_rate": 4.169741697416975e-06,
+      "loss": 1.604,
+      "step": 616
+    },
+    {
+      "epoch": 0.7678904791537026,
+      "grad_norm": 2.5792219638824463,
+      "learning_rate": 4.1674354243542434e-06,
+      "loss": 1.5976,
+      "step": 617
+    },
+    {
+      "epoch": 0.7691350342252644,
+      "grad_norm": 2.845416307449341,
+      "learning_rate": 4.165129151291513e-06,
+      "loss": 1.5945,
+      "step": 618
+    },
+    {
+      "epoch": 0.7703795892968264,
+      "grad_norm": 2.8647871017456055,
+      "learning_rate": 4.162822878228783e-06,
+      "loss": 1.5637,
+      "step": 619
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "grad_norm": 2.590719699859619,
+      "learning_rate": 4.1605166051660526e-06,
+      "loss": 1.556,
+      "step": 620
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "eval_loss": 1.7115222215652466,
+      "eval_runtime": 50.3905,
+      "eval_samples_per_second": 19.845,
+      "eval_steps_per_second": 0.833,
+      "step": 620
+    },
+    {
+      "epoch": 0.7728686994399502,
+      "grad_norm": 2.626723527908325,
+      "learning_rate": 4.158210332103321e-06,
+      "loss": 1.5486,
+      "step": 621
+    },
+    {
+      "epoch": 0.7741132545115121,
+      "grad_norm": 2.713069438934326,
+      "learning_rate": 4.1559040590405906e-06,
+      "loss": 1.5474,
+      "step": 622
+    },
+    {
+      "epoch": 0.7753578095830741,
+      "grad_norm": 2.685563087463379,
+      "learning_rate": 4.15359778597786e-06,
+      "loss": 1.5529,
+      "step": 623
+    },
+    {
+      "epoch": 0.776602364654636,
+      "grad_norm": 2.690586805343628,
+      "learning_rate": 4.151291512915129e-06,
+      "loss": 1.5293,
+      "step": 624
+    },
+    {
+      "epoch": 0.7778469197261979,
+      "grad_norm": 2.865345001220703,
+      "learning_rate": 4.148985239852399e-06,
+      "loss": 1.5095,
+      "step": 625
+    },
+    {
+      "epoch": 0.7790914747977598,
+      "grad_norm": 2.92393159866333,
+      "learning_rate": 4.146678966789668e-06,
+      "loss": 1.5561,
+      "step": 626
+    },
+    {
+      "epoch": 0.7803360298693217,
+      "grad_norm": 2.554480791091919,
+      "learning_rate": 4.144372693726938e-06,
+      "loss": 1.4876,
+      "step": 627
+    },
+    {
+      "epoch": 0.7815805849408837,
+      "grad_norm": 2.6128833293914795,
+      "learning_rate": 4.142066420664207e-06,
+      "loss": 1.5378,
+      "step": 628
+    },
+    {
+      "epoch": 0.7828251400124455,
+      "grad_norm": 3.0110888481140137,
+      "learning_rate": 4.1397601476014765e-06,
+      "loss": 1.5859,
+      "step": 629
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "grad_norm": 2.8152706623077393,
+      "learning_rate": 4.137453874538745e-06,
+      "loss": 1.5841,
+      "step": 630
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "eval_loss": 1.702181339263916,
+      "eval_runtime": 53.6926,
+      "eval_samples_per_second": 18.625,
+      "eval_steps_per_second": 0.782,
+      "step": 630
+    },
+    {
+      "epoch": 0.7853142501555694,
+      "grad_norm": 2.8634233474731445,
+      "learning_rate": 4.135147601476015e-06,
+      "loss": 1.564,
+      "step": 631
+    },
+    {
+      "epoch": 0.7865588052271313,
+      "grad_norm": 2.8939666748046875,
+      "learning_rate": 4.132841328413285e-06,
+      "loss": 1.5859,
+      "step": 632
+    },
+    {
+      "epoch": 0.7878033602986932,
+      "grad_norm": 2.6560444831848145,
+      "learning_rate": 4.130535055350554e-06,
+      "loss": 1.5199,
+      "step": 633
+    },
+    {
+      "epoch": 0.7890479153702551,
+      "grad_norm": 2.964721202850342,
+      "learning_rate": 4.128228782287823e-06,
+      "loss": 1.5888,
+      "step": 634
+    },
+    {
+      "epoch": 0.790292470441817,
+      "grad_norm": 2.74668288230896,
+      "learning_rate": 4.125922509225092e-06,
+      "loss": 1.5847,
+      "step": 635
+    },
+    {
+      "epoch": 0.791537025513379,
+      "grad_norm": 2.723123550415039,
+      "learning_rate": 4.1236162361623625e-06,
+      "loss": 1.5166,
+      "step": 636
+    },
+    {
+      "epoch": 0.7927815805849409,
+      "grad_norm": 2.7265713214874268,
+      "learning_rate": 4.121309963099631e-06,
+      "loss": 1.5954,
+      "step": 637
+    },
+    {
+      "epoch": 0.7940261356565028,
+      "grad_norm": 2.979126214981079,
+      "learning_rate": 4.1190036900369005e-06,
+      "loss": 1.5583,
+      "step": 638
+    },
+    {
+      "epoch": 0.7952706907280647,
+      "grad_norm": 2.844376802444458,
+      "learning_rate": 4.11669741697417e-06,
+      "loss": 1.5564,
+      "step": 639
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "grad_norm": 2.7583703994750977,
+      "learning_rate": 4.114391143911439e-06,
+      "loss": 1.5094,
+      "step": 640
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "eval_loss": 1.6986565589904785,
+      "eval_runtime": 47.3549,
+      "eval_samples_per_second": 21.117,
+      "eval_steps_per_second": 0.887,
+      "step": 640
+    },
+    {
+      "epoch": 0.7977598008711886,
+      "grad_norm": 2.8942224979400635,
+      "learning_rate": 4.112084870848709e-06,
+      "loss": 1.5504,
+      "step": 641
+    },
+    {
+      "epoch": 0.7990043559427504,
+      "grad_norm": 2.6590495109558105,
+      "learning_rate": 4.109778597785978e-06,
+      "loss": 1.5227,
+      "step": 642
+    },
+    {
+      "epoch": 0.8002489110143124,
+      "grad_norm": 2.5988378524780273,
+      "learning_rate": 4.107472324723248e-06,
+      "loss": 1.5054,
+      "step": 643
+    },
+    {
+      "epoch": 0.8014934660858744,
+      "grad_norm": 2.787335157394409,
+      "learning_rate": 4.105166051660517e-06,
+      "loss": 1.5844,
+      "step": 644
+    },
+    {
+      "epoch": 0.8027380211574362,
+      "grad_norm": 2.8202896118164062,
+      "learning_rate": 4.1028597785977864e-06,
+      "loss": 1.5596,
+      "step": 645
+    },
+    {
+      "epoch": 0.8039825762289982,
+      "grad_norm": 2.65376615524292,
+      "learning_rate": 4.100553505535056e-06,
+      "loss": 1.5893,
+      "step": 646
+    },
+    {
+      "epoch": 0.80522713130056,
+      "grad_norm": 2.5933308601379395,
+      "learning_rate": 4.098247232472325e-06,
+      "loss": 1.5403,
+      "step": 647
+    },
+    {
+      "epoch": 0.806471686372122,
+      "grad_norm": 2.777070999145508,
+      "learning_rate": 4.095940959409595e-06,
+      "loss": 1.5231,
+      "step": 648
+    },
+    {
+      "epoch": 0.8077162414436839,
+      "grad_norm": 2.6427664756774902,
+      "learning_rate": 4.093634686346864e-06,
+      "loss": 1.5937,
+      "step": 649
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "grad_norm": 2.651561737060547,
+      "learning_rate": 4.091328413284133e-06,
+      "loss": 1.4847,
+      "step": 650
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "eval_loss": 1.7089996337890625,
+      "eval_runtime": 47.4346,
+      "eval_samples_per_second": 21.082,
+      "eval_steps_per_second": 0.885,
+      "step": 650
+    },
+    {
+      "epoch": 0.8102053515868077,
+      "grad_norm": 2.844067335128784,
+      "learning_rate": 4.089022140221402e-06,
+      "loss": 1.488,
+      "step": 651
+    },
+    {
+      "epoch": 0.8114499066583696,
+      "grad_norm": 2.603480577468872,
+      "learning_rate": 4.086715867158672e-06,
+      "loss": 1.5248,
+      "step": 652
+    },
+    {
+      "epoch": 0.8126944617299315,
+      "grad_norm": 2.789322853088379,
+      "learning_rate": 4.084409594095941e-06,
+      "loss": 1.5119,
+      "step": 653
+    },
+    {
+      "epoch": 0.8139390168014935,
+      "grad_norm": 2.7183773517608643,
+      "learning_rate": 4.08210332103321e-06,
+      "loss": 1.5123,
+      "step": 654
+    },
+    {
+      "epoch": 0.8151835718730553,
+      "grad_norm": 2.80613374710083,
+      "learning_rate": 4.07979704797048e-06,
+      "loss": 1.5526,
+      "step": 655
+    },
+    {
+      "epoch": 0.8164281269446173,
+      "grad_norm": 2.641035795211792,
+      "learning_rate": 4.077490774907749e-06,
+      "loss": 1.6127,
+      "step": 656
+    },
+    {
+      "epoch": 0.8176726820161793,
+      "grad_norm": 2.773684024810791,
+      "learning_rate": 4.075184501845019e-06,
+      "loss": 1.5404,
+      "step": 657
+    },
+    {
+      "epoch": 0.8189172370877411,
+      "grad_norm": 2.6951382160186768,
+      "learning_rate": 4.072878228782288e-06,
+      "loss": 1.4928,
+      "step": 658
+    },
+    {
+      "epoch": 0.8201617921593031,
+      "grad_norm": 2.931771755218506,
+      "learning_rate": 4.0705719557195575e-06,
+      "loss": 1.5949,
+      "step": 659
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "grad_norm": 2.8786075115203857,
+      "learning_rate": 4.068265682656827e-06,
+      "loss": 1.5582,
+      "step": 660
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "eval_loss": 1.698564052581787,
+      "eval_runtime": 46.6299,
+      "eval_samples_per_second": 21.445,
+      "eval_steps_per_second": 0.901,
+      "step": 660
+    },
+    {
+      "epoch": 0.8226509023024269,
+      "grad_norm": 2.7402451038360596,
+      "learning_rate": 4.065959409594096e-06,
+      "loss": 1.5562,
+      "step": 661
+    },
+    {
+      "epoch": 0.8238954573739888,
+      "grad_norm": 2.7968692779541016,
+      "learning_rate": 4.063653136531366e-06,
+      "loss": 1.556,
+      "step": 662
+    },
+    {
+      "epoch": 0.8251400124455507,
+      "grad_norm": 2.8233132362365723,
+      "learning_rate": 4.061346863468635e-06,
+      "loss": 1.5527,
+      "step": 663
+    },
+    {
+      "epoch": 0.8263845675171126,
+      "grad_norm": 2.713390827178955,
+      "learning_rate": 4.059040590405905e-06,
+      "loss": 1.6012,
+      "step": 664
+    },
+    {
+      "epoch": 0.8276291225886746,
+      "grad_norm": 2.7783989906311035,
+      "learning_rate": 4.056734317343174e-06,
+      "loss": 1.5247,
+      "step": 665
+    },
+    {
+      "epoch": 0.8288736776602365,
+      "grad_norm": 2.808554172515869,
+      "learning_rate": 4.054428044280443e-06,
+      "loss": 1.5412,
+      "step": 666
+    },
+    {
+      "epoch": 0.8301182327317984,
+      "grad_norm": 2.8476831912994385,
+      "learning_rate": 4.052121771217712e-06,
+      "loss": 1.5307,
+      "step": 667
+    },
+    {
+      "epoch": 0.8313627878033603,
+      "grad_norm": 2.8287601470947266,
+      "learning_rate": 4.049815498154982e-06,
+      "loss": 1.6021,
+      "step": 668
+    },
+    {
+      "epoch": 0.8326073428749222,
+      "grad_norm": 2.975510835647583,
+      "learning_rate": 4.047509225092252e-06,
+      "loss": 1.5474,
+      "step": 669
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "grad_norm": 2.723236560821533,
+      "learning_rate": 4.04520295202952e-06,
+      "loss": 1.5242,
+      "step": 670
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "eval_loss": 1.7005239725112915,
+      "eval_runtime": 49.4504,
+      "eval_samples_per_second": 20.222,
+      "eval_steps_per_second": 0.849,
+      "step": 670
+    },
+    {
+      "epoch": 0.835096453018046,
+      "grad_norm": 2.7521748542785645,
+      "learning_rate": 4.04289667896679e-06,
+      "loss": 1.5467,
+      "step": 671
+    },
+    {
+      "epoch": 0.836341008089608,
+      "grad_norm": 2.7770044803619385,
+      "learning_rate": 4.04059040590406e-06,
+      "loss": 1.5407,
+      "step": 672
+    },
+    {
+      "epoch": 0.8375855631611698,
+      "grad_norm": 2.744323492050171,
+      "learning_rate": 4.038284132841329e-06,
+      "loss": 1.5422,
+      "step": 673
+    },
+    {
+      "epoch": 0.8388301182327318,
+      "grad_norm": 2.6699817180633545,
+      "learning_rate": 4.035977859778598e-06,
+      "loss": 1.5508,
+      "step": 674
+    },
+    {
+      "epoch": 0.8400746733042938,
+      "grad_norm": 2.705273151397705,
+      "learning_rate": 4.0336715867158674e-06,
+      "loss": 1.5099,
+      "step": 675
+    },
+    {
+      "epoch": 0.8413192283758556,
+      "grad_norm": 2.9485747814178467,
+      "learning_rate": 4.031365313653137e-06,
+      "loss": 1.5529,
+      "step": 676
+    },
+    {
+      "epoch": 0.8425637834474176,
+      "grad_norm": 3.1750423908233643,
+      "learning_rate": 4.029059040590406e-06,
+      "loss": 1.5894,
+      "step": 677
+    },
+    {
+      "epoch": 0.8438083385189795,
+      "grad_norm": 2.76448655128479,
+      "learning_rate": 4.026752767527676e-06,
+      "loss": 1.5752,
+      "step": 678
+    },
+    {
+      "epoch": 0.8450528935905414,
+      "grad_norm": 2.676708698272705,
+      "learning_rate": 4.024446494464945e-06,
+      "loss": 1.5487,
+      "step": 679
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "grad_norm": 2.6832051277160645,
+      "learning_rate": 4.0221402214022145e-06,
+      "loss": 1.5055,
+      "step": 680
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "eval_loss": 1.6945017576217651,
+      "eval_runtime": 43.9369,
+      "eval_samples_per_second": 22.76,
+      "eval_steps_per_second": 0.956,
+      "step": 680
+    },
+    {
+      "epoch": 0.8475420037336652,
+      "grad_norm": 2.6812336444854736,
+      "learning_rate": 4.019833948339484e-06,
+      "loss": 1.5315,
+      "step": 681
+    },
+    {
+      "epoch": 0.8487865588052271,
+      "grad_norm": 2.9394326210021973,
+      "learning_rate": 4.017527675276753e-06,
+      "loss": 1.5636,
+      "step": 682
+    },
+    {
+      "epoch": 0.8500311138767891,
+      "grad_norm": 2.944952964782715,
+      "learning_rate": 4.015221402214022e-06,
+      "loss": 1.56,
+      "step": 683
+    },
+    {
+      "epoch": 0.8512756689483509,
+      "grad_norm": 2.7475314140319824,
+      "learning_rate": 4.012915129151292e-06,
+      "loss": 1.5442,
+      "step": 684
+    },
+    {
+      "epoch": 0.8525202240199129,
+      "grad_norm": 3.0572879314422607,
+      "learning_rate": 4.010608856088562e-06,
+      "loss": 1.5023,
+      "step": 685
+    },
+    {
+      "epoch": 0.8537647790914747,
+      "grad_norm": 2.75365948677063,
+      "learning_rate": 4.00830258302583e-06,
+      "loss": 1.5414,
+      "step": 686
+    },
+    {
+      "epoch": 0.8550093341630367,
+      "grad_norm": 2.5714449882507324,
+      "learning_rate": 4.0059963099631e-06,
+      "loss": 1.5027,
+      "step": 687
+    },
+    {
+      "epoch": 0.8562538892345987,
+      "grad_norm": 2.755167007446289,
+      "learning_rate": 4.003690036900369e-06,
+      "loss": 1.5788,
+      "step": 688
+    },
+    {
+      "epoch": 0.8574984443061605,
+      "grad_norm": 2.798967123031616,
+      "learning_rate": 4.0013837638376385e-06,
+      "loss": 1.5162,
+      "step": 689
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "grad_norm": 2.803614854812622,
+      "learning_rate": 3.999077490774908e-06,
+      "loss": 1.5079,
+      "step": 690
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "eval_loss": 1.6924962997436523,
+      "eval_runtime": 44.47,
+      "eval_samples_per_second": 22.487,
+      "eval_steps_per_second": 0.944,
+      "step": 690
+    },
+    {
+      "epoch": 0.8599875544492844,
+      "grad_norm": 2.6898066997528076,
+      "learning_rate": 3.996771217712177e-06,
+      "loss": 1.5478,
+      "step": 691
+    },
+    {
+      "epoch": 0.8612321095208463,
+      "grad_norm": 2.6376144886016846,
+      "learning_rate": 3.994464944649447e-06,
+      "loss": 1.5362,
+      "step": 692
+    },
+    {
+      "epoch": 0.8624766645924082,
+      "grad_norm": 2.7240405082702637,
+      "learning_rate": 3.992158671586716e-06,
+      "loss": 1.5502,
+      "step": 693
+    },
+    {
+      "epoch": 0.8637212196639701,
+      "grad_norm": 2.929445266723633,
+      "learning_rate": 3.989852398523986e-06,
+      "loss": 1.5799,
+      "step": 694
+    },
+    {
+      "epoch": 0.864965774735532,
+      "grad_norm": 2.593223810195923,
+      "learning_rate": 3.987546125461255e-06,
+      "loss": 1.5352,
+      "step": 695
+    },
+    {
+      "epoch": 0.866210329807094,
+      "grad_norm": 2.7710154056549072,
+      "learning_rate": 3.9852398523985245e-06,
+      "loss": 1.5048,
+      "step": 696
+    },
+    {
+      "epoch": 0.8674548848786559,
+      "grad_norm": 2.7370848655700684,
+      "learning_rate": 3.982933579335794e-06,
+      "loss": 1.5233,
+      "step": 697
+    },
+    {
+      "epoch": 0.8686994399502178,
+      "grad_norm": 2.8632168769836426,
+      "learning_rate": 3.980627306273063e-06,
+      "loss": 1.5435,
+      "step": 698
+    },
+    {
+      "epoch": 0.8699439950217797,
+      "grad_norm": 2.752298593521118,
+      "learning_rate": 3.978321033210332e-06,
+      "loss": 1.5728,
+      "step": 699
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "grad_norm": 2.8345625400543213,
+      "learning_rate": 3.976014760147602e-06,
+      "loss": 1.5727,
+      "step": 700
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "eval_loss": 1.6855305433273315,
+      "eval_runtime": 43.3683,
+      "eval_samples_per_second": 23.058,
+      "eval_steps_per_second": 0.968,
+      "step": 700
+    },
+    {
+      "epoch": 0.8724331051649036,
+      "grad_norm": 2.8135335445404053,
+      "learning_rate": 3.973708487084872e-06,
+      "loss": 1.5086,
+      "step": 701
+    },
+    {
+      "epoch": 0.8736776602364654,
+      "grad_norm": 2.7416603565216064,
+      "learning_rate": 3.97140221402214e-06,
+      "loss": 1.545,
+      "step": 702
+    },
+    {
+      "epoch": 0.8749222153080274,
+      "grad_norm": 2.6902289390563965,
+      "learning_rate": 3.96909594095941e-06,
+      "loss": 1.5509,
+      "step": 703
+    },
+    {
+      "epoch": 0.8761667703795893,
+      "grad_norm": 2.6537084579467773,
+      "learning_rate": 3.966789667896679e-06,
+      "loss": 1.562,
+      "step": 704
+    },
+    {
+      "epoch": 0.8774113254511512,
+      "grad_norm": 2.6979284286499023,
+      "learning_rate": 3.964483394833948e-06,
+      "loss": 1.4829,
+      "step": 705
+    },
+    {
+      "epoch": 0.8786558805227132,
+      "grad_norm": 2.5795764923095703,
+      "learning_rate": 3.962177121771218e-06,
+      "loss": 1.4677,
+      "step": 706
+    },
+    {
+      "epoch": 0.879900435594275,
+      "grad_norm": 3.0185086727142334,
+      "learning_rate": 3.959870848708487e-06,
+      "loss": 1.5608,
+      "step": 707
+    },
+    {
+      "epoch": 0.881144990665837,
+      "grad_norm": 2.6458113193511963,
+      "learning_rate": 3.957564575645757e-06,
+      "loss": 1.5342,
+      "step": 708
+    },
+    {
+      "epoch": 0.8823895457373989,
+      "grad_norm": 2.7985665798187256,
+      "learning_rate": 3.955258302583026e-06,
+      "loss": 1.527,
+      "step": 709
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "grad_norm": 2.9069082736968994,
+      "learning_rate": 3.9529520295202955e-06,
+      "loss": 1.5156,
+      "step": 710
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "eval_loss": 1.6898822784423828,
+      "eval_runtime": 42.8749,
+      "eval_samples_per_second": 23.324,
+      "eval_steps_per_second": 0.98,
+      "step": 710
+    },
+    {
+      "epoch": 0.8848786558805227,
+      "grad_norm": 2.760143995285034,
+      "learning_rate": 3.950645756457565e-06,
+      "loss": 1.5124,
+      "step": 711
+    },
+    {
+      "epoch": 0.8861232109520847,
+      "grad_norm": 2.8085529804229736,
+      "learning_rate": 3.948339483394834e-06,
+      "loss": 1.5862,
+      "step": 712
+    },
+    {
+      "epoch": 0.8873677660236465,
+      "grad_norm": 2.909905195236206,
+      "learning_rate": 3.946033210332104e-06,
+      "loss": 1.5557,
+      "step": 713
+    },
+    {
+      "epoch": 0.8886123210952085,
+      "grad_norm": 2.826899290084839,
+      "learning_rate": 3.943726937269373e-06,
+      "loss": 1.5277,
+      "step": 714
+    },
+    {
+      "epoch": 0.8898568761667703,
+      "grad_norm": 2.6269052028656006,
+      "learning_rate": 3.941420664206642e-06,
+      "loss": 1.4819,
+      "step": 715
+    },
+    {
+      "epoch": 0.8911014312383323,
+      "grad_norm": 2.6687698364257812,
+      "learning_rate": 3.939114391143912e-06,
+      "loss": 1.5156,
+      "step": 716
+    },
+    {
+      "epoch": 0.8923459863098943,
+      "grad_norm": 2.9605629444122314,
+      "learning_rate": 3.9368081180811815e-06,
+      "loss": 1.5358,
+      "step": 717
+    },
+    {
+      "epoch": 0.8935905413814561,
+      "grad_norm": 3.0367166996002197,
+      "learning_rate": 3.934501845018451e-06,
+      "loss": 1.5847,
+      "step": 718
+    },
+    {
+      "epoch": 0.8948350964530181,
+      "grad_norm": 2.528796434402466,
+      "learning_rate": 3.9321955719557195e-06,
+      "loss": 1.4851,
+      "step": 719
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "grad_norm": 2.7116613388061523,
+      "learning_rate": 3.929889298892989e-06,
+      "loss": 1.4706,
+      "step": 720
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "eval_loss": 1.6845883131027222,
+      "eval_runtime": 44.4734,
+      "eval_samples_per_second": 22.485,
+      "eval_steps_per_second": 0.944,
+      "step": 720
+    },
+    {
+      "epoch": 0.8973242065961419,
+      "grad_norm": 2.771864414215088,
+      "learning_rate": 3.927583025830259e-06,
+      "loss": 1.5444,
+      "step": 721
+    },
+    {
+      "epoch": 0.8985687616677038,
+      "grad_norm": 2.773746967315674,
+      "learning_rate": 3.925276752767528e-06,
+      "loss": 1.564,
+      "step": 722
+    },
+    {
+      "epoch": 0.8998133167392657,
+      "grad_norm": 2.7069854736328125,
+      "learning_rate": 3.922970479704797e-06,
+      "loss": 1.5601,
+      "step": 723
+    },
+    {
+      "epoch": 0.9010578718108276,
+      "grad_norm": 2.840261697769165,
+      "learning_rate": 3.920664206642067e-06,
+      "loss": 1.5733,
+      "step": 724
+    },
+    {
+      "epoch": 0.9023024268823896,
+      "grad_norm": 2.709897041320801,
+      "learning_rate": 3.918357933579336e-06,
+      "loss": 1.528,
+      "step": 725
+    },
+    {
+      "epoch": 0.9035469819539514,
+      "grad_norm": 2.662367343902588,
+      "learning_rate": 3.9160516605166055e-06,
+      "loss": 1.5532,
+      "step": 726
+    },
+    {
+      "epoch": 0.9047915370255134,
+      "grad_norm": 2.6775717735290527,
+      "learning_rate": 3.913745387453875e-06,
+      "loss": 1.5464,
+      "step": 727
+    },
+    {
+      "epoch": 0.9060360920970753,
+      "grad_norm": 2.617842674255371,
+      "learning_rate": 3.911439114391144e-06,
+      "loss": 1.4539,
+      "step": 728
+    },
+    {
+      "epoch": 0.9072806471686372,
+      "grad_norm": 2.668649435043335,
+      "learning_rate": 3.909132841328414e-06,
+      "loss": 1.5627,
+      "step": 729
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "grad_norm": 2.5331950187683105,
+      "learning_rate": 3.906826568265683e-06,
+      "loss": 1.5165,
+      "step": 730
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "eval_loss": 1.6860331296920776,
+      "eval_runtime": 46.094,
+      "eval_samples_per_second": 21.695,
+      "eval_steps_per_second": 0.911,
+      "step": 730
+    },
+    {
+      "epoch": 0.909769757311761,
+      "grad_norm": 2.602492332458496,
+      "learning_rate": 3.9045202952029526e-06,
+      "loss": 1.5145,
+      "step": 731
+    },
+    {
+      "epoch": 0.911014312383323,
+      "grad_norm": 2.632782459259033,
+      "learning_rate": 3.902214022140222e-06,
+      "loss": 1.4413,
+      "step": 732
+    },
+    {
+      "epoch": 0.9122588674548848,
+      "grad_norm": 2.6025912761688232,
+      "learning_rate": 3.899907749077491e-06,
+      "loss": 1.5263,
+      "step": 733
+    },
+    {
+      "epoch": 0.9135034225264468,
+      "grad_norm": 2.770116090774536,
+      "learning_rate": 3.897601476014761e-06,
+      "loss": 1.5514,
+      "step": 734
+    },
+    {
+      "epoch": 0.9147479775980087,
+      "grad_norm": 2.7822961807250977,
+      "learning_rate": 3.895295202952029e-06,
+      "loss": 1.5296,
+      "step": 735
+    },
+    {
+      "epoch": 0.9159925326695706,
+      "grad_norm": 2.6689720153808594,
+      "learning_rate": 3.892988929889299e-06,
+      "loss": 1.4552,
+      "step": 736
+    },
+    {
+      "epoch": 0.9172370877411326,
+      "grad_norm": 2.8021798133850098,
+      "learning_rate": 3.890682656826569e-06,
+      "loss": 1.5308,
+      "step": 737
+    },
+    {
+      "epoch": 0.9184816428126945,
+      "grad_norm": 2.575313091278076,
+      "learning_rate": 3.888376383763838e-06,
+      "loss": 1.4807,
+      "step": 738
+    },
+    {
+      "epoch": 0.9197261978842564,
+      "grad_norm": 2.5955779552459717,
+      "learning_rate": 3.886070110701107e-06,
+      "loss": 1.4222,
+      "step": 739
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "grad_norm": 2.647939920425415,
+      "learning_rate": 3.8837638376383765e-06,
+      "loss": 1.5285,
+      "step": 740
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "eval_loss": 1.6907480955123901,
+      "eval_runtime": 45.7678,
+      "eval_samples_per_second": 21.849,
+      "eval_steps_per_second": 0.918,
+      "step": 740
+    },
+    {
+      "epoch": 0.9222153080273802,
+      "grad_norm": 2.7664523124694824,
+      "learning_rate": 3.881457564575646e-06,
+      "loss": 1.5349,
+      "step": 741
+    },
+    {
+      "epoch": 0.9234598630989421,
+      "grad_norm": 2.5770998001098633,
+      "learning_rate": 3.879151291512915e-06,
+      "loss": 1.5071,
+      "step": 742
+    },
+    {
+      "epoch": 0.9247044181705041,
+      "grad_norm": 2.502567768096924,
+      "learning_rate": 3.876845018450185e-06,
+      "loss": 1.492,
+      "step": 743
+    },
+    {
+      "epoch": 0.9259489732420659,
+      "grad_norm": 2.6531119346618652,
+      "learning_rate": 3.874538745387454e-06,
+      "loss": 1.5045,
+      "step": 744
+    },
+    {
+      "epoch": 0.9271935283136279,
+      "grad_norm": 2.7343876361846924,
+      "learning_rate": 3.872232472324724e-06,
+      "loss": 1.5194,
+      "step": 745
+    },
+    {
+      "epoch": 0.9284380833851898,
+      "grad_norm": 2.842313528060913,
+      "learning_rate": 3.869926199261993e-06,
+      "loss": 1.5588,
+      "step": 746
+    },
+    {
+      "epoch": 0.9296826384567517,
+      "grad_norm": 2.718245267868042,
+      "learning_rate": 3.8676199261992625e-06,
+      "loss": 1.5252,
+      "step": 747
+    },
+    {
+      "epoch": 0.9309271935283137,
+      "grad_norm": 2.695392370223999,
+      "learning_rate": 3.865313653136532e-06,
+      "loss": 1.5596,
+      "step": 748
+    },
+    {
+      "epoch": 0.9321717485998755,
+      "grad_norm": 2.7859959602355957,
+      "learning_rate": 3.863007380073801e-06,
+      "loss": 1.5151,
+      "step": 749
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "grad_norm": 2.5199174880981445,
+      "learning_rate": 3.860701107011071e-06,
+      "loss": 1.4948,
+      "step": 750
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "eval_loss": 1.6868674755096436,
+      "eval_runtime": 46.4115,
+      "eval_samples_per_second": 21.546,
+      "eval_steps_per_second": 0.905,
+      "step": 750
+    },
+    {
+      "epoch": 0.9346608587429994,
+      "grad_norm": 2.8194046020507812,
+      "learning_rate": 3.858394833948339e-06,
+      "loss": 1.5144,
+      "step": 751
+    },
+    {
+      "epoch": 0.9359054138145613,
+      "grad_norm": 2.810380697250366,
+      "learning_rate": 3.856088560885609e-06,
+      "loss": 1.4998,
+      "step": 752
+    },
+    {
+      "epoch": 0.9371499688861232,
+      "grad_norm": 2.6906750202178955,
+      "learning_rate": 3.853782287822879e-06,
+      "loss": 1.4748,
+      "step": 753
+    },
+    {
+      "epoch": 0.9383945239576851,
+      "grad_norm": 2.9030277729034424,
+      "learning_rate": 3.851476014760148e-06,
+      "loss": 1.615,
+      "step": 754
+    },
+    {
+      "epoch": 0.939639079029247,
+      "grad_norm": 2.8690176010131836,
+      "learning_rate": 3.849169741697417e-06,
+      "loss": 1.5083,
+      "step": 755
+    },
+    {
+      "epoch": 0.940883634100809,
+      "grad_norm": 2.669646978378296,
+      "learning_rate": 3.8468634686346865e-06,
+      "loss": 1.5572,
+      "step": 756
+    },
+    {
+      "epoch": 0.9421281891723708,
+      "grad_norm": 2.6517696380615234,
+      "learning_rate": 3.844557195571956e-06,
+      "loss": 1.5142,
+      "step": 757
+    },
+    {
+      "epoch": 0.9433727442439328,
+      "grad_norm": 2.5524444580078125,
+      "learning_rate": 3.842250922509225e-06,
+      "loss": 1.4924,
+      "step": 758
+    },
+    {
+      "epoch": 0.9446172993154948,
+      "grad_norm": 2.6532633304595947,
+      "learning_rate": 3.839944649446495e-06,
+      "loss": 1.484,
+      "step": 759
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "grad_norm": 2.7779057025909424,
+      "learning_rate": 3.837638376383764e-06,
+      "loss": 1.5106,
+      "step": 760
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "eval_loss": 1.6845752000808716,
+      "eval_runtime": 50.3415,
+      "eval_samples_per_second": 19.864,
+      "eval_steps_per_second": 0.834,
+      "step": 760
+    },
+    {
+      "epoch": 0.9471064094586186,
+      "grad_norm": 2.6192541122436523,
+      "learning_rate": 3.8353321033210336e-06,
+      "loss": 1.5454,
+      "step": 761
+    },
+    {
+      "epoch": 0.9483509645301804,
+      "grad_norm": 2.561861991882324,
+      "learning_rate": 3.833025830258303e-06,
+      "loss": 1.5141,
+      "step": 762
+    },
+    {
+      "epoch": 0.9495955196017424,
+      "grad_norm": 2.661829948425293,
+      "learning_rate": 3.830719557195572e-06,
+      "loss": 1.4997,
+      "step": 763
+    },
+    {
+      "epoch": 0.9508400746733043,
+      "grad_norm": 2.725275754928589,
+      "learning_rate": 3.828413284132842e-06,
+      "loss": 1.5224,
+      "step": 764
+    },
+    {
+      "epoch": 0.9520846297448662,
+      "grad_norm": 2.5664193630218506,
+      "learning_rate": 3.826107011070111e-06,
+      "loss": 1.5073,
+      "step": 765
+    },
+    {
+      "epoch": 0.9533291848164281,
+      "grad_norm": 2.595189332962036,
+      "learning_rate": 3.823800738007381e-06,
+      "loss": 1.5379,
+      "step": 766
+    },
+    {
+      "epoch": 0.95457373988799,
+      "grad_norm": 2.738060235977173,
+      "learning_rate": 3.821494464944649e-06,
+      "loss": 1.4814,
+      "step": 767
+    },
+    {
+      "epoch": 0.955818294959552,
+      "grad_norm": 2.601071357727051,
+      "learning_rate": 3.819188191881919e-06,
+      "loss": 1.4812,
+      "step": 768
+    },
+    {
+      "epoch": 0.9570628500311139,
+      "grad_norm": 2.657212257385254,
+      "learning_rate": 3.816881918819189e-06,
+      "loss": 1.4748,
+      "step": 769
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "grad_norm": 2.7673351764678955,
+      "learning_rate": 3.814575645756458e-06,
+      "loss": 1.5535,
+      "step": 770
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "eval_loss": 1.677243947982788,
+      "eval_runtime": 51.5537,
+      "eval_samples_per_second": 19.397,
+      "eval_steps_per_second": 0.815,
+      "step": 770
+    },
+    {
+      "epoch": 0.9595519601742377,
+      "grad_norm": 2.760890007019043,
+      "learning_rate": 3.812269372693727e-06,
+      "loss": 1.5654,
+      "step": 771
+    },
+    {
+      "epoch": 0.9607965152457997,
+      "grad_norm": 2.6934309005737305,
+      "learning_rate": 3.809963099630997e-06,
+      "loss": 1.5149,
+      "step": 772
+    },
+    {
+      "epoch": 0.9620410703173615,
+      "grad_norm": 2.729950428009033,
+      "learning_rate": 3.8076568265682662e-06,
+      "loss": 1.5579,
+      "step": 773
+    },
+    {
+      "epoch": 0.9632856253889235,
+      "grad_norm": 2.732926607131958,
+      "learning_rate": 3.8053505535055352e-06,
+      "loss": 1.5179,
+      "step": 774
+    },
+    {
+      "epoch": 0.9645301804604853,
+      "grad_norm": 2.745391368865967,
+      "learning_rate": 3.8030442804428046e-06,
+      "loss": 1.5124,
+      "step": 775
+    },
+    {
+      "epoch": 0.9657747355320473,
+      "grad_norm": 2.8050146102905273,
+      "learning_rate": 3.800738007380074e-06,
+      "loss": 1.519,
+      "step": 776
+    },
+    {
+      "epoch": 0.9670192906036092,
+      "grad_norm": 2.697171449661255,
+      "learning_rate": 3.798431734317343e-06,
+      "loss": 1.5219,
+      "step": 777
+    },
+    {
+      "epoch": 0.9682638456751711,
+      "grad_norm": 2.6375980377197266,
+      "learning_rate": 3.796125461254613e-06,
+      "loss": 1.5345,
+      "step": 778
+    },
+    {
+      "epoch": 0.9695084007467331,
+      "grad_norm": 2.586636781692505,
+      "learning_rate": 3.7938191881918823e-06,
+      "loss": 1.5182,
+      "step": 779
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "grad_norm": 2.626453399658203,
+      "learning_rate": 3.7915129151291518e-06,
+      "loss": 1.4817,
+      "step": 780
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "eval_loss": 1.6734713315963745,
+      "eval_runtime": 48.1436,
+      "eval_samples_per_second": 20.771,
+      "eval_steps_per_second": 0.872,
+      "step": 780
+    },
+    {
+      "epoch": 0.9719975108898569,
+      "grad_norm": 2.579970121383667,
+      "learning_rate": 3.7892066420664208e-06,
+      "loss": 1.4981,
+      "step": 781
+    },
+    {
+      "epoch": 0.9732420659614188,
+      "grad_norm": 2.6834843158721924,
+      "learning_rate": 3.7869003690036906e-06,
+      "loss": 1.5044,
+      "step": 782
+    },
+    {
+      "epoch": 0.9744866210329807,
+      "grad_norm": 2.586982488632202,
+      "learning_rate": 3.78459409594096e-06,
+      "loss": 1.5079,
+      "step": 783
+    },
+    {
+      "epoch": 0.9757311761045426,
+      "grad_norm": 2.910027503967285,
+      "learning_rate": 3.782287822878229e-06,
+      "loss": 1.5695,
+      "step": 784
+    },
+    {
+      "epoch": 0.9769757311761046,
+      "grad_norm": 2.7088494300842285,
+      "learning_rate": 3.7799815498154984e-06,
+      "loss": 1.4986,
+      "step": 785
+    },
+    {
+      "epoch": 0.9782202862476664,
+      "grad_norm": 2.581325054168701,
+      "learning_rate": 3.777675276752768e-06,
+      "loss": 1.5137,
+      "step": 786
+    },
+    {
+      "epoch": 0.9794648413192284,
+      "grad_norm": 2.700709819793701,
+      "learning_rate": 3.775369003690037e-06,
+      "loss": 1.4718,
+      "step": 787
+    },
+    {
+      "epoch": 0.9807093963907902,
+      "grad_norm": 2.6712708473205566,
+      "learning_rate": 3.7730627306273067e-06,
+      "loss": 1.5208,
+      "step": 788
+    },
+    {
+      "epoch": 0.9819539514623522,
+      "grad_norm": 2.6799817085266113,
+      "learning_rate": 3.770756457564576e-06,
+      "loss": 1.5374,
+      "step": 789
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "grad_norm": 2.618988037109375,
+      "learning_rate": 3.768450184501845e-06,
+      "loss": 1.4997,
+      "step": 790
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "eval_loss": 1.6860820055007935,
+      "eval_runtime": 51.7654,
+      "eval_samples_per_second": 19.318,
+      "eval_steps_per_second": 0.811,
+      "step": 790
+    },
+    {
+      "epoch": 0.984443061605476,
+      "grad_norm": 2.6899526119232178,
+      "learning_rate": 3.7661439114391146e-06,
+      "loss": 1.5216,
+      "step": 791
+    },
+    {
+      "epoch": 0.985687616677038,
+      "grad_norm": 2.8003487586975098,
+      "learning_rate": 3.763837638376384e-06,
+      "loss": 1.557,
+      "step": 792
+    },
+    {
+      "epoch": 0.9869321717485999,
+      "grad_norm": 2.744536876678467,
+      "learning_rate": 3.761531365313654e-06,
+      "loss": 1.5355,
+      "step": 793
+    },
+    {
+      "epoch": 0.9881767268201618,
+      "grad_norm": 2.587250232696533,
+      "learning_rate": 3.759225092250923e-06,
+      "loss": 1.5259,
+      "step": 794
+    },
+    {
+      "epoch": 0.9894212818917237,
+      "grad_norm": 2.616291046142578,
+      "learning_rate": 3.7569188191881922e-06,
+      "loss": 1.5176,
+      "step": 795
+    },
+    {
+      "epoch": 0.9906658369632856,
+      "grad_norm": 2.6410577297210693,
+      "learning_rate": 3.7546125461254617e-06,
+      "loss": 1.511,
+      "step": 796
+    },
+    {
+      "epoch": 0.9919103920348475,
+      "grad_norm": 2.577373504638672,
+      "learning_rate": 3.7523062730627307e-06,
+      "loss": 1.4715,
+      "step": 797
+    },
+    {
+      "epoch": 0.9931549471064095,
+      "grad_norm": 2.67305326461792,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 1.4828,
+      "step": 798
+    },
+    {
+      "epoch": 0.9943995021779714,
+      "grad_norm": 3.1631500720977783,
+      "learning_rate": 3.74769372693727e-06,
+      "loss": 1.5602,
+      "step": 799
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "grad_norm": 2.9222350120544434,
+      "learning_rate": 3.745387453874539e-06,
+      "loss": 1.4814,
+      "step": 800
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "eval_loss": 1.6810544729232788,
+      "eval_runtime": 51.1262,
+      "eval_samples_per_second": 19.559,
+      "eval_steps_per_second": 0.821,
+      "step": 800
+    },
+    {
+      "epoch": 0.9968886123210952,
+      "grad_norm": 2.884312629699707,
+      "learning_rate": 3.7430811808118084e-06,
+      "loss": 1.4553,
+      "step": 801
+    },
+    {
+      "epoch": 0.9981331673926571,
+      "grad_norm": 2.712655782699585,
+      "learning_rate": 3.7407749077490778e-06,
+      "loss": 1.5275,
+      "step": 802
+    },
+    {
+      "epoch": 0.9993777224642191,
+      "grad_norm": 2.8700571060180664,
+      "learning_rate": 3.7384686346863468e-06,
+      "loss": 1.4923,
+      "step": 803
+    },
+    {
+      "epoch": 1.000622277535781,
+      "grad_norm": 2.7779245376586914,
+      "learning_rate": 3.7361623616236166e-06,
+      "loss": 1.5224,
+      "step": 804
+    },
+    {
+      "epoch": 1.0018668326073428,
+      "grad_norm": 2.7679810523986816,
+      "learning_rate": 3.733856088560886e-06,
+      "loss": 1.4977,
+      "step": 805
+    },
+    {
+      "epoch": 1.0031113876789048,
+      "grad_norm": 2.797023296356201,
+      "learning_rate": 3.7315498154981555e-06,
+      "loss": 1.4396,
+      "step": 806
+    },
+    {
+      "epoch": 1.0043559427504667,
+      "grad_norm": 3.007962465286255,
+      "learning_rate": 3.7292435424354245e-06,
+      "loss": 1.5015,
+      "step": 807
+    },
+    {
+      "epoch": 1.0056004978220285,
+      "grad_norm": 3.127639055252075,
+      "learning_rate": 3.726937269372694e-06,
+      "loss": 1.5176,
+      "step": 808
+    },
+    {
+      "epoch": 1.0068450528935906,
+      "grad_norm": 2.9001357555389404,
+      "learning_rate": 3.7246309963099637e-06,
+      "loss": 1.4735,
+      "step": 809
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "grad_norm": 2.8245413303375244,
+      "learning_rate": 3.7223247232472327e-06,
+      "loss": 1.486,
+      "step": 810
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "eval_loss": 1.6835130453109741,
+      "eval_runtime": 44.5682,
+      "eval_samples_per_second": 22.438,
+      "eval_steps_per_second": 0.942,
+      "step": 810
+    },
+    {
+      "epoch": 1.0093341630367143,
+      "grad_norm": 2.780172824859619,
+      "learning_rate": 3.720018450184502e-06,
+      "loss": 1.4437,
+      "step": 811
+    },
+    {
+      "epoch": 1.0105787181082764,
+      "grad_norm": 2.6013996601104736,
+      "learning_rate": 3.7177121771217716e-06,
+      "loss": 1.4529,
+      "step": 812
+    },
+    {
+      "epoch": 1.0118232731798382,
+      "grad_norm": 2.831015110015869,
+      "learning_rate": 3.7154059040590406e-06,
+      "loss": 1.4825,
+      "step": 813
+    },
+    {
+      "epoch": 1.0130678282514,
+      "grad_norm": 2.8341829776763916,
+      "learning_rate": 3.71309963099631e-06,
+      "loss": 1.4736,
+      "step": 814
+    },
+    {
+      "epoch": 1.0143123833229621,
+      "grad_norm": 2.769540309906006,
+      "learning_rate": 3.71079335793358e-06,
+      "loss": 1.4724,
+      "step": 815
+    },
+    {
+      "epoch": 1.015556938394524,
+      "grad_norm": 3.0083069801330566,
+      "learning_rate": 3.708487084870849e-06,
+      "loss": 1.4985,
+      "step": 816
+    },
+    {
+      "epoch": 1.0168014934660858,
+      "grad_norm": 3.052168607711792,
+      "learning_rate": 3.7061808118081183e-06,
+      "loss": 1.5193,
+      "step": 817
+    },
+    {
+      "epoch": 1.0180460485376477,
+      "grad_norm": 2.924877405166626,
+      "learning_rate": 3.7038745387453877e-06,
+      "loss": 1.5219,
+      "step": 818
+    },
+    {
+      "epoch": 1.0192906036092098,
+      "grad_norm": 3.150123357772827,
+      "learning_rate": 3.7015682656826576e-06,
+      "loss": 1.5026,
+      "step": 819
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "grad_norm": 2.881655216217041,
+      "learning_rate": 3.6992619926199266e-06,
+      "loss": 1.5011,
+      "step": 820
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "eval_loss": 1.6745303869247437,
+      "eval_runtime": 50.6791,
+      "eval_samples_per_second": 19.732,
+      "eval_steps_per_second": 0.829,
+      "step": 820
+    },
+    {
+      "epoch": 1.0217797137523335,
+      "grad_norm": 2.824249505996704,
+      "learning_rate": 3.696955719557196e-06,
+      "loss": 1.4642,
+      "step": 821
+    },
+    {
+      "epoch": 1.0230242688238955,
+      "grad_norm": 2.859144926071167,
+      "learning_rate": 3.6946494464944654e-06,
+      "loss": 1.4681,
+      "step": 822
+    },
+    {
+      "epoch": 1.0242688238954574,
+      "grad_norm": 2.90312123298645,
+      "learning_rate": 3.6923431734317344e-06,
+      "loss": 1.4926,
+      "step": 823
+    },
+    {
+      "epoch": 1.0255133789670192,
+      "grad_norm": 2.8340659141540527,
+      "learning_rate": 3.690036900369004e-06,
+      "loss": 1.4834,
+      "step": 824
+    },
+    {
+      "epoch": 1.0267579340385813,
+      "grad_norm": 2.9743151664733887,
+      "learning_rate": 3.6877306273062737e-06,
+      "loss": 1.4574,
+      "step": 825
+    },
+    {
+      "epoch": 1.0280024891101431,
+      "grad_norm": 3.032179594039917,
+      "learning_rate": 3.6854243542435427e-06,
+      "loss": 1.5571,
+      "step": 826
+    },
+    {
+      "epoch": 1.029247044181705,
+      "grad_norm": 2.816826343536377,
+      "learning_rate": 3.683118081180812e-06,
+      "loss": 1.455,
+      "step": 827
+    },
+    {
+      "epoch": 1.030491599253267,
+      "grad_norm": 3.046696186065674,
+      "learning_rate": 3.6808118081180815e-06,
+      "loss": 1.4871,
+      "step": 828
+    },
+    {
+      "epoch": 1.031736154324829,
+      "grad_norm": 2.8517391681671143,
+      "learning_rate": 3.678505535055351e-06,
+      "loss": 1.5292,
+      "step": 829
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "grad_norm": 2.752811908721924,
+      "learning_rate": 3.67619926199262e-06,
+      "loss": 1.5091,
+      "step": 830
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "eval_loss": 1.6692287921905518,
+      "eval_runtime": 50.8347,
+      "eval_samples_per_second": 19.672,
+      "eval_steps_per_second": 0.826,
+      "step": 830
+    },
+    {
+      "epoch": 1.0342252644679526,
+      "grad_norm": 2.9220356941223145,
+      "learning_rate": 3.6738929889298898e-06,
+      "loss": 1.4247,
+      "step": 831
+    },
+    {
+      "epoch": 1.0354698195395147,
+      "grad_norm": 2.8903002738952637,
+      "learning_rate": 3.671586715867159e-06,
+      "loss": 1.5595,
+      "step": 832
+    },
+    {
+      "epoch": 1.0367143746110765,
+      "grad_norm": 2.6495542526245117,
+      "learning_rate": 3.669280442804428e-06,
+      "loss": 1.4076,
+      "step": 833
+    },
+    {
+      "epoch": 1.0379589296826384,
+      "grad_norm": 2.880809783935547,
+      "learning_rate": 3.6669741697416976e-06,
+      "loss": 1.5285,
+      "step": 834
+    },
+    {
+      "epoch": 1.0392034847542004,
+      "grad_norm": 2.919142007827759,
+      "learning_rate": 3.6646678966789675e-06,
+      "loss": 1.5165,
+      "step": 835
+    },
+    {
+      "epoch": 1.0404480398257623,
+      "grad_norm": 3.1180522441864014,
+      "learning_rate": 3.6623616236162365e-06,
+      "loss": 1.5224,
+      "step": 836
+    },
+    {
+      "epoch": 1.0416925948973241,
+      "grad_norm": 2.767709255218506,
+      "learning_rate": 3.660055350553506e-06,
+      "loss": 1.4859,
+      "step": 837
+    },
+    {
+      "epoch": 1.0429371499688862,
+      "grad_norm": 2.708828926086426,
+      "learning_rate": 3.6577490774907753e-06,
+      "loss": 1.4803,
+      "step": 838
+    },
+    {
+      "epoch": 1.044181705040448,
+      "grad_norm": 2.688359260559082,
+      "learning_rate": 3.6554428044280443e-06,
+      "loss": 1.4571,
+      "step": 839
+    },
+    {
+      "epoch": 1.04542626011201,
+      "grad_norm": 2.691812753677368,
+      "learning_rate": 3.6531365313653137e-06,
+      "loss": 1.5043,
+      "step": 840
+    },
+    {
+      "epoch": 1.04542626011201,
+      "eval_loss": 1.6769559383392334,
+      "eval_runtime": 49.6233,
+      "eval_samples_per_second": 20.152,
+      "eval_steps_per_second": 0.846,
+      "step": 840
+    },
+    {
+      "epoch": 1.046670815183572,
+      "grad_norm": 2.686372995376587,
+      "learning_rate": 3.6508302583025836e-06,
+      "loss": 1.5221,
+      "step": 841
+    },
+    {
+      "epoch": 1.0479153702551338,
+      "grad_norm": 2.742859125137329,
+      "learning_rate": 3.648523985239853e-06,
+      "loss": 1.4685,
+      "step": 842
+    },
+    {
+      "epoch": 1.0491599253266957,
+      "grad_norm": 2.874018907546997,
+      "learning_rate": 3.646217712177122e-06,
+      "loss": 1.4813,
+      "step": 843
+    },
+    {
+      "epoch": 1.0504044803982575,
+      "grad_norm": 2.8159990310668945,
+      "learning_rate": 3.6439114391143914e-06,
+      "loss": 1.4421,
+      "step": 844
+    },
+    {
+      "epoch": 1.0516490354698196,
+      "grad_norm": 2.714787006378174,
+      "learning_rate": 3.641605166051661e-06,
+      "loss": 1.4879,
+      "step": 845
+    },
+    {
+      "epoch": 1.0528935905413814,
+      "grad_norm": 2.885148525238037,
+      "learning_rate": 3.63929889298893e-06,
+      "loss": 1.4725,
+      "step": 846
+    },
+    {
+      "epoch": 1.0541381456129433,
+      "grad_norm": 2.7272610664367676,
+      "learning_rate": 3.6369926199261997e-06,
+      "loss": 1.4276,
+      "step": 847
+    },
+    {
+      "epoch": 1.0553827006845053,
+      "grad_norm": 2.9114978313446045,
+      "learning_rate": 3.634686346863469e-06,
+      "loss": 1.4194,
+      "step": 848
+    },
+    {
+      "epoch": 1.0566272557560672,
+      "grad_norm": 3.019313335418701,
+      "learning_rate": 3.632380073800738e-06,
+      "loss": 1.5266,
+      "step": 849
+    },
+    {
+      "epoch": 1.057871810827629,
+      "grad_norm": 2.714751958847046,
+      "learning_rate": 3.6300738007380075e-06,
+      "loss": 1.4593,
+      "step": 850
+    },
+    {
+      "epoch": 1.057871810827629,
+      "eval_loss": 1.6760298013687134,
+      "eval_runtime": 52.1681,
+      "eval_samples_per_second": 19.169,
+      "eval_steps_per_second": 0.805,
+      "step": 850
+    },
+    {
+      "epoch": 1.0591163658991911,
+      "grad_norm": 3.0507936477661133,
+      "learning_rate": 3.627767527675277e-06,
+      "loss": 1.5064,
+      "step": 851
+    },
+    {
+      "epoch": 1.060360920970753,
+      "grad_norm": 2.8116295337677,
+      "learning_rate": 3.625461254612546e-06,
+      "loss": 1.5218,
+      "step": 852
+    },
+    {
+      "epoch": 1.0616054760423148,
+      "grad_norm": 2.8801465034484863,
+      "learning_rate": 3.623154981549816e-06,
+      "loss": 1.4641,
+      "step": 853
+    },
+    {
+      "epoch": 1.0628500311138769,
+      "grad_norm": 3.0581862926483154,
+      "learning_rate": 3.6208487084870852e-06,
+      "loss": 1.483,
+      "step": 854
+    },
+    {
+      "epoch": 1.0640945861854387,
+      "grad_norm": 2.898780584335327,
+      "learning_rate": 3.6185424354243547e-06,
+      "loss": 1.4623,
+      "step": 855
+    },
+    {
+      "epoch": 1.0653391412570006,
+      "grad_norm": 2.971820116043091,
+      "learning_rate": 3.6162361623616237e-06,
+      "loss": 1.4455,
+      "step": 856
+    },
+    {
+      "epoch": 1.0665836963285624,
+      "grad_norm": 3.0021913051605225,
+      "learning_rate": 3.6139298892988935e-06,
+      "loss": 1.4878,
+      "step": 857
+    },
+    {
+      "epoch": 1.0678282514001245,
+      "grad_norm": 3.0697457790374756,
+      "learning_rate": 3.611623616236163e-06,
+      "loss": 1.5008,
+      "step": 858
+    },
+    {
+      "epoch": 1.0690728064716863,
+      "grad_norm": 3.0787923336029053,
+      "learning_rate": 3.609317343173432e-06,
+      "loss": 1.4612,
+      "step": 859
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "grad_norm": 2.789339780807495,
+      "learning_rate": 3.6070110701107014e-06,
+      "loss": 1.4287,
+      "step": 860
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "eval_loss": 1.6747506856918335,
+      "eval_runtime": 51.4912,
+      "eval_samples_per_second": 19.421,
+      "eval_steps_per_second": 0.816,
+      "step": 860
+    },
+    {
+      "epoch": 1.0715619166148103,
+      "grad_norm": 2.9622788429260254,
+      "learning_rate": 3.6047047970479708e-06,
+      "loss": 1.4905,
+      "step": 861
+    },
+    {
+      "epoch": 1.072806471686372,
+      "grad_norm": 3.0675268173217773,
+      "learning_rate": 3.6023985239852398e-06,
+      "loss": 1.461,
+      "step": 862
+    },
+    {
+      "epoch": 1.074051026757934,
+      "grad_norm": 2.820996046066284,
+      "learning_rate": 3.6000922509225096e-06,
+      "loss": 1.4948,
+      "step": 863
+    },
+    {
+      "epoch": 1.075295581829496,
+      "grad_norm": 2.833761215209961,
+      "learning_rate": 3.597785977859779e-06,
+      "loss": 1.4683,
+      "step": 864
+    },
+    {
+      "epoch": 1.0765401369010579,
+      "grad_norm": 2.917313814163208,
+      "learning_rate": 3.595479704797048e-06,
+      "loss": 1.5191,
+      "step": 865
+    },
+    {
+      "epoch": 1.0777846919726197,
+      "grad_norm": 2.876166820526123,
+      "learning_rate": 3.5931734317343175e-06,
+      "loss": 1.4854,
+      "step": 866
+    },
+    {
+      "epoch": 1.0790292470441818,
+      "grad_norm": 3.139049768447876,
+      "learning_rate": 3.590867158671587e-06,
+      "loss": 1.5134,
+      "step": 867
+    },
+    {
+      "epoch": 1.0802738021157436,
+      "grad_norm": 2.9956753253936768,
+      "learning_rate": 3.5885608856088567e-06,
+      "loss": 1.5003,
+      "step": 868
+    },
+    {
+      "epoch": 1.0815183571873055,
+      "grad_norm": 2.957181930541992,
+      "learning_rate": 3.5862546125461257e-06,
+      "loss": 1.4588,
+      "step": 869
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "grad_norm": 2.8576438426971436,
+      "learning_rate": 3.583948339483395e-06,
+      "loss": 1.48,
+      "step": 870
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "eval_loss": 1.6724848747253418,
+      "eval_runtime": 50.9669,
+      "eval_samples_per_second": 19.621,
+      "eval_steps_per_second": 0.824,
+      "step": 870
+    },
+    {
+      "epoch": 1.0840074673304294,
+      "grad_norm": 2.8183560371398926,
+      "learning_rate": 3.5816420664206646e-06,
+      "loss": 1.4818,
+      "step": 871
+    },
+    {
+      "epoch": 1.0852520224019913,
+      "grad_norm": 2.7531797885894775,
+      "learning_rate": 3.5793357933579336e-06,
+      "loss": 1.4712,
+      "step": 872
+    },
+    {
+      "epoch": 1.086496577473553,
+      "grad_norm": 2.923962354660034,
+      "learning_rate": 3.5770295202952034e-06,
+      "loss": 1.4864,
+      "step": 873
+    },
+    {
+      "epoch": 1.0877411325451152,
+      "grad_norm": 2.851724147796631,
+      "learning_rate": 3.574723247232473e-06,
+      "loss": 1.4771,
+      "step": 874
+    },
+    {
+      "epoch": 1.088985687616677,
+      "grad_norm": 3.024634838104248,
+      "learning_rate": 3.572416974169742e-06,
+      "loss": 1.4643,
+      "step": 875
+    },
+    {
+      "epoch": 1.0902302426882389,
+      "grad_norm": 2.7100844383239746,
+      "learning_rate": 3.5701107011070113e-06,
+      "loss": 1.4773,
+      "step": 876
+    },
+    {
+      "epoch": 1.091474797759801,
+      "grad_norm": 2.730502128601074,
+      "learning_rate": 3.5678044280442807e-06,
+      "loss": 1.5051,
+      "step": 877
+    },
+    {
+      "epoch": 1.0927193528313628,
+      "grad_norm": 2.747082233428955,
+      "learning_rate": 3.5654981549815497e-06,
+      "loss": 1.5207,
+      "step": 878
+    },
+    {
+      "epoch": 1.0939639079029246,
+      "grad_norm": 2.718358039855957,
+      "learning_rate": 3.5631918819188195e-06,
+      "loss": 1.5067,
+      "step": 879
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "grad_norm": 2.7794573307037354,
+      "learning_rate": 3.560885608856089e-06,
+      "loss": 1.4616,
+      "step": 880
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "eval_loss": 1.6676132678985596,
+      "eval_runtime": 51.6157,
+      "eval_samples_per_second": 19.374,
+      "eval_steps_per_second": 0.814,
+      "step": 880
+    },
+    {
+      "epoch": 1.0964530180460486,
+      "grad_norm": 2.727416753768921,
+      "learning_rate": 3.5585793357933584e-06,
+      "loss": 1.4043,
+      "step": 881
+    },
+    {
+      "epoch": 1.0976975731176104,
+      "grad_norm": 2.856900930404663,
+      "learning_rate": 3.5562730627306274e-06,
+      "loss": 1.5198,
+      "step": 882
+    },
+    {
+      "epoch": 1.0989421281891725,
+      "grad_norm": 2.872823715209961,
+      "learning_rate": 3.553966789667897e-06,
+      "loss": 1.4981,
+      "step": 883
+    },
+    {
+      "epoch": 1.1001866832607343,
+      "grad_norm": 2.7724361419677734,
+      "learning_rate": 3.5516605166051667e-06,
+      "loss": 1.4819,
+      "step": 884
+    },
+    {
+      "epoch": 1.1014312383322962,
+      "grad_norm": 2.884434461593628,
+      "learning_rate": 3.5493542435424357e-06,
+      "loss": 1.433,
+      "step": 885
+    },
+    {
+      "epoch": 1.102675793403858,
+      "grad_norm": 2.887873411178589,
+      "learning_rate": 3.547047970479705e-06,
+      "loss": 1.4677,
+      "step": 886
+    },
+    {
+      "epoch": 1.10392034847542,
+      "grad_norm": 2.946845054626465,
+      "learning_rate": 3.5447416974169745e-06,
+      "loss": 1.4983,
+      "step": 887
+    },
+    {
+      "epoch": 1.105164903546982,
+      "grad_norm": 2.752552032470703,
+      "learning_rate": 3.5424354243542435e-06,
+      "loss": 1.4766,
+      "step": 888
+    },
+    {
+      "epoch": 1.1064094586185438,
+      "grad_norm": 2.788634777069092,
+      "learning_rate": 3.5401291512915133e-06,
+      "loss": 1.4747,
+      "step": 889
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "grad_norm": 2.782792568206787,
+      "learning_rate": 3.5378228782287828e-06,
+      "loss": 1.5503,
+      "step": 890
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "eval_loss": 1.6698333024978638,
+      "eval_runtime": 49.1373,
+      "eval_samples_per_second": 20.351,
+      "eval_steps_per_second": 0.855,
+      "step": 890
+    },
+    {
+      "epoch": 1.1088985687616677,
+      "grad_norm": 2.8351166248321533,
+      "learning_rate": 3.535516605166052e-06,
+      "loss": 1.4589,
+      "step": 891
+    },
+    {
+      "epoch": 1.1101431238332296,
+      "grad_norm": 2.798872709274292,
+      "learning_rate": 3.533210332103321e-06,
+      "loss": 1.4496,
+      "step": 892
+    },
+    {
+      "epoch": 1.1113876789047916,
+      "grad_norm": 3.0863239765167236,
+      "learning_rate": 3.5309040590405906e-06,
+      "loss": 1.4698,
+      "step": 893
+    },
+    {
+      "epoch": 1.1126322339763535,
+      "grad_norm": 2.853862762451172,
+      "learning_rate": 3.5285977859778605e-06,
+      "loss": 1.4429,
+      "step": 894
+    },
+    {
+      "epoch": 1.1138767890479153,
+      "grad_norm": 2.8946170806884766,
+      "learning_rate": 3.5262915129151295e-06,
+      "loss": 1.4697,
+      "step": 895
+    },
+    {
+      "epoch": 1.1151213441194774,
+      "grad_norm": 3.043823003768921,
+      "learning_rate": 3.523985239852399e-06,
+      "loss": 1.4666,
+      "step": 896
+    },
+    {
+      "epoch": 1.1163658991910392,
+      "grad_norm": 2.7822980880737305,
+      "learning_rate": 3.5216789667896683e-06,
+      "loss": 1.4523,
+      "step": 897
+    },
+    {
+      "epoch": 1.117610454262601,
+      "grad_norm": 2.9454171657562256,
+      "learning_rate": 3.5193726937269373e-06,
+      "loss": 1.5009,
+      "step": 898
+    },
+    {
+      "epoch": 1.1188550093341632,
+      "grad_norm": 2.8533740043640137,
+      "learning_rate": 3.5170664206642067e-06,
+      "loss": 1.4265,
+      "step": 899
+    },
+    {
+      "epoch": 1.120099564405725,
+      "grad_norm": 2.894937038421631,
+      "learning_rate": 3.5147601476014766e-06,
+      "loss": 1.4895,
+      "step": 900
+    },
+    {
+      "epoch": 1.120099564405725,
+      "eval_loss": 1.6701407432556152,
+      "eval_runtime": 45.477,
+      "eval_samples_per_second": 21.989,
+      "eval_steps_per_second": 0.924,
+      "step": 900
+    },
+    {
+      "epoch": 1.1213441194772868,
+      "grad_norm": 2.933821201324463,
+      "learning_rate": 3.5124538745387456e-06,
+      "loss": 1.468,
+      "step": 901
+    },
+    {
+      "epoch": 1.1225886745488487,
+      "grad_norm": 2.834265947341919,
+      "learning_rate": 3.510147601476015e-06,
+      "loss": 1.4621,
+      "step": 902
+    },
+    {
+      "epoch": 1.1238332296204108,
+      "grad_norm": 2.9064722061157227,
+      "learning_rate": 3.5078413284132844e-06,
+      "loss": 1.504,
+      "step": 903
+    },
+    {
+      "epoch": 1.1250777846919726,
+      "grad_norm": 2.8753886222839355,
+      "learning_rate": 3.505535055350554e-06,
+      "loss": 1.4643,
+      "step": 904
+    },
+    {
+      "epoch": 1.1263223397635345,
+      "grad_norm": 2.894659996032715,
+      "learning_rate": 3.503228782287823e-06,
+      "loss": 1.4909,
+      "step": 905
+    },
+    {
+      "epoch": 1.1275668948350965,
+      "grad_norm": 3.085977792739868,
+      "learning_rate": 3.5009225092250927e-06,
+      "loss": 1.459,
+      "step": 906
+    },
+    {
+      "epoch": 1.1288114499066584,
+      "grad_norm": 2.9958064556121826,
+      "learning_rate": 3.498616236162362e-06,
+      "loss": 1.4152,
+      "step": 907
+    },
+    {
+      "epoch": 1.1300560049782202,
+      "grad_norm": 2.9136762619018555,
+      "learning_rate": 3.496309963099631e-06,
+      "loss": 1.473,
+      "step": 908
+    },
+    {
+      "epoch": 1.1313005600497823,
+      "grad_norm": 2.956437826156616,
+      "learning_rate": 3.4940036900369005e-06,
+      "loss": 1.5167,
+      "step": 909
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "grad_norm": 2.7354865074157715,
+      "learning_rate": 3.4916974169741704e-06,
+      "loss": 1.4413,
+      "step": 910
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "eval_loss": 1.6629912853240967,
+      "eval_runtime": 41.5478,
+      "eval_samples_per_second": 24.069,
+      "eval_steps_per_second": 1.011,
+      "step": 910
+    },
+    {
+      "epoch": 1.133789670192906,
+      "grad_norm": 2.7443008422851562,
+      "learning_rate": 3.4893911439114394e-06,
+      "loss": 1.4475,
+      "step": 911
+    },
+    {
+      "epoch": 1.135034225264468,
+      "grad_norm": 2.7811238765716553,
+      "learning_rate": 3.487084870848709e-06,
+      "loss": 1.4257,
+      "step": 912
+    },
+    {
+      "epoch": 1.13627878033603,
+      "grad_norm": 2.7916479110717773,
+      "learning_rate": 3.4847785977859782e-06,
+      "loss": 1.4048,
+      "step": 913
+    },
+    {
+      "epoch": 1.1375233354075918,
+      "grad_norm": 2.7835633754730225,
+      "learning_rate": 3.4824723247232472e-06,
+      "loss": 1.3996,
+      "step": 914
+    },
+    {
+      "epoch": 1.1387678904791536,
+      "grad_norm": 2.8188705444335938,
+      "learning_rate": 3.4801660516605166e-06,
+      "loss": 1.4541,
+      "step": 915
+    },
+    {
+      "epoch": 1.1400124455507157,
+      "grad_norm": 3.0134100914001465,
+      "learning_rate": 3.4778597785977865e-06,
+      "loss": 1.4353,
+      "step": 916
+    },
+    {
+      "epoch": 1.1412570006222775,
+      "grad_norm": 2.6965606212615967,
+      "learning_rate": 3.475553505535056e-06,
+      "loss": 1.4308,
+      "step": 917
+    },
+    {
+      "epoch": 1.1425015556938394,
+      "grad_norm": 2.8179919719696045,
+      "learning_rate": 3.473247232472325e-06,
+      "loss": 1.5204,
+      "step": 918
+    },
+    {
+      "epoch": 1.1437461107654014,
+      "grad_norm": 2.7213902473449707,
+      "learning_rate": 3.4709409594095943e-06,
+      "loss": 1.4344,
+      "step": 919
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "grad_norm": 2.850193738937378,
+      "learning_rate": 3.4686346863468638e-06,
+      "loss": 1.473,
+      "step": 920
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "eval_loss": 1.663345217704773,
+      "eval_runtime": 43.8397,
+      "eval_samples_per_second": 22.81,
+      "eval_steps_per_second": 0.958,
+      "step": 920
+    },
+    {
+      "epoch": 1.1462352209085251,
+      "grad_norm": 2.780198574066162,
+      "learning_rate": 3.4663284132841328e-06,
+      "loss": 1.4472,
+      "step": 921
+    },
+    {
+      "epoch": 1.1474797759800872,
+      "grad_norm": 2.9373831748962402,
+      "learning_rate": 3.4640221402214026e-06,
+      "loss": 1.426,
+      "step": 922
+    },
+    {
+      "epoch": 1.148724331051649,
+      "grad_norm": 2.6986873149871826,
+      "learning_rate": 3.461715867158672e-06,
+      "loss": 1.3953,
+      "step": 923
+    },
+    {
+      "epoch": 1.149968886123211,
+      "grad_norm": 2.927441120147705,
+      "learning_rate": 3.459409594095941e-06,
+      "loss": 1.4578,
+      "step": 924
+    },
+    {
+      "epoch": 1.151213441194773,
+      "grad_norm": 2.8469648361206055,
+      "learning_rate": 3.4571033210332105e-06,
+      "loss": 1.4313,
+      "step": 925
+    },
+    {
+      "epoch": 1.1524579962663348,
+      "grad_norm": 2.8114013671875,
+      "learning_rate": 3.4547970479704803e-06,
+      "loss": 1.4267,
+      "step": 926
+    },
+    {
+      "epoch": 1.1537025513378967,
+      "grad_norm": 2.977605104446411,
+      "learning_rate": 3.4524907749077493e-06,
+      "loss": 1.4747,
+      "step": 927
+    },
+    {
+      "epoch": 1.1549471064094585,
+      "grad_norm": 2.9327852725982666,
+      "learning_rate": 3.4501845018450187e-06,
+      "loss": 1.4219,
+      "step": 928
+    },
+    {
+      "epoch": 1.1561916614810206,
+      "grad_norm": 2.844038963317871,
+      "learning_rate": 3.447878228782288e-06,
+      "loss": 1.5146,
+      "step": 929
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "grad_norm": 2.9402201175689697,
+      "learning_rate": 3.4455719557195576e-06,
+      "loss": 1.45,
+      "step": 930
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "eval_loss": 1.6620122194290161,
+      "eval_runtime": 46.1748,
+      "eval_samples_per_second": 21.657,
+      "eval_steps_per_second": 0.91,
+      "step": 930
+    },
+    {
+      "epoch": 1.1586807716241443,
+      "grad_norm": 2.842651128768921,
+      "learning_rate": 3.4432656826568266e-06,
+      "loss": 1.4415,
+      "step": 931
+    },
+    {
+      "epoch": 1.1599253266957064,
+      "grad_norm": 2.8909685611724854,
+      "learning_rate": 3.4409594095940964e-06,
+      "loss": 1.4529,
+      "step": 932
+    },
+    {
+      "epoch": 1.1611698817672682,
+      "grad_norm": 2.8025808334350586,
+      "learning_rate": 3.438653136531366e-06,
+      "loss": 1.4552,
+      "step": 933
+    },
+    {
+      "epoch": 1.16241443683883,
+      "grad_norm": 2.6995794773101807,
+      "learning_rate": 3.436346863468635e-06,
+      "loss": 1.4435,
+      "step": 934
+    },
+    {
+      "epoch": 1.1636589919103921,
+      "grad_norm": 2.7307121753692627,
+      "learning_rate": 3.4340405904059043e-06,
+      "loss": 1.4447,
+      "step": 935
+    },
+    {
+      "epoch": 1.164903546981954,
+      "grad_norm": 2.9532501697540283,
+      "learning_rate": 3.4317343173431737e-06,
+      "loss": 1.4747,
+      "step": 936
+    },
+    {
+      "epoch": 1.1661481020535158,
+      "grad_norm": 3.0135788917541504,
+      "learning_rate": 3.4294280442804427e-06,
+      "loss": 1.525,
+      "step": 937
+    },
+    {
+      "epoch": 1.167392657125078,
+      "grad_norm": 2.796355962753296,
+      "learning_rate": 3.4271217712177125e-06,
+      "loss": 1.4539,
+      "step": 938
+    },
+    {
+      "epoch": 1.1686372121966397,
+      "grad_norm": 2.9422309398651123,
+      "learning_rate": 3.424815498154982e-06,
+      "loss": 1.4664,
+      "step": 939
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "grad_norm": 2.7680039405822754,
+      "learning_rate": 3.4225092250922514e-06,
+      "loss": 1.4837,
+      "step": 940
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "eval_loss": 1.660908818244934,
+      "eval_runtime": 45.6998,
+      "eval_samples_per_second": 21.882,
+      "eval_steps_per_second": 0.919,
+      "step": 940
+    },
+    {
+      "epoch": 1.1711263223397634,
+      "grad_norm": 2.8888845443725586,
+      "learning_rate": 3.4202029520295204e-06,
+      "loss": 1.5315,
+      "step": 941
+    },
+    {
+      "epoch": 1.1723708774113255,
+      "grad_norm": 2.7124130725860596,
+      "learning_rate": 3.41789667896679e-06,
+      "loss": 1.4188,
+      "step": 942
+    },
+    {
+      "epoch": 1.1736154324828874,
+      "grad_norm": 2.75203013420105,
+      "learning_rate": 3.4155904059040596e-06,
+      "loss": 1.4533,
+      "step": 943
+    },
+    {
+      "epoch": 1.1748599875544492,
+      "grad_norm": 2.865480422973633,
+      "learning_rate": 3.4132841328413286e-06,
+      "loss": 1.4532,
+      "step": 944
+    },
+    {
+      "epoch": 1.1761045426260113,
+      "grad_norm": 2.8618321418762207,
+      "learning_rate": 3.410977859778598e-06,
+      "loss": 1.4386,
+      "step": 945
+    },
+    {
+      "epoch": 1.1773490976975731,
+      "grad_norm": 2.7656145095825195,
+      "learning_rate": 3.4086715867158675e-06,
+      "loss": 1.3996,
+      "step": 946
+    },
+    {
+      "epoch": 1.178593652769135,
+      "grad_norm": 2.76213002204895,
+      "learning_rate": 3.4063653136531365e-06,
+      "loss": 1.4979,
+      "step": 947
+    },
+    {
+      "epoch": 1.179838207840697,
+      "grad_norm": 2.8581957817077637,
+      "learning_rate": 3.4040590405904063e-06,
+      "loss": 1.4673,
+      "step": 948
+    },
+    {
+      "epoch": 1.1810827629122589,
+      "grad_norm": 2.7825586795806885,
+      "learning_rate": 3.4017527675276758e-06,
+      "loss": 1.4453,
+      "step": 949
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "grad_norm": 2.9226131439208984,
+      "learning_rate": 3.3994464944649448e-06,
+      "loss": 1.4879,
+      "step": 950
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "eval_loss": 1.6621263027191162,
+      "eval_runtime": 45.1222,
+      "eval_samples_per_second": 22.162,
+      "eval_steps_per_second": 0.931,
+      "step": 950
+    },
+    {
+      "epoch": 1.1835718730553828,
+      "grad_norm": 2.9997975826263428,
+      "learning_rate": 3.397140221402214e-06,
+      "loss": 1.505,
+      "step": 951
+    },
+    {
+      "epoch": 1.1848164281269447,
+      "grad_norm": 2.7225453853607178,
+      "learning_rate": 3.3948339483394836e-06,
+      "loss": 1.4695,
+      "step": 952
+    },
+    {
+      "epoch": 1.1860609831985065,
+      "grad_norm": 2.916473150253296,
+      "learning_rate": 3.3925276752767534e-06,
+      "loss": 1.4901,
+      "step": 953
+    },
+    {
+      "epoch": 1.1873055382700684,
+      "grad_norm": 2.8371644020080566,
+      "learning_rate": 3.3902214022140224e-06,
+      "loss": 1.4944,
+      "step": 954
+    },
+    {
+      "epoch": 1.1885500933416304,
+      "grad_norm": 2.813662528991699,
+      "learning_rate": 3.387915129151292e-06,
+      "loss": 1.4603,
+      "step": 955
+    },
+    {
+      "epoch": 1.1897946484131923,
+      "grad_norm": 2.991931676864624,
+      "learning_rate": 3.3856088560885613e-06,
+      "loss": 1.5496,
+      "step": 956
+    },
+    {
+      "epoch": 1.1910392034847541,
+      "grad_norm": 2.6252501010894775,
+      "learning_rate": 3.3833025830258303e-06,
+      "loss": 1.4405,
+      "step": 957
+    },
+    {
+      "epoch": 1.1922837585563162,
+      "grad_norm": 2.747725009918213,
+      "learning_rate": 3.3809963099630997e-06,
+      "loss": 1.4689,
+      "step": 958
+    },
+    {
+      "epoch": 1.193528313627878,
+      "grad_norm": 2.887763261795044,
+      "learning_rate": 3.3786900369003696e-06,
+      "loss": 1.4501,
+      "step": 959
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "grad_norm": 2.9954354763031006,
+      "learning_rate": 3.3763837638376386e-06,
+      "loss": 1.4978,
+      "step": 960
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "eval_loss": 1.664442539215088,
+      "eval_runtime": 49.1266,
+      "eval_samples_per_second": 20.356,
+      "eval_steps_per_second": 0.855,
+      "step": 960
+    },
+    {
+      "epoch": 1.196017423771002,
+      "grad_norm": 3.0174152851104736,
+      "learning_rate": 3.374077490774908e-06,
+      "loss": 1.4353,
+      "step": 961
+    },
+    {
+      "epoch": 1.1972619788425638,
+      "grad_norm": 2.9082882404327393,
+      "learning_rate": 3.3717712177121774e-06,
+      "loss": 1.4878,
+      "step": 962
+    },
+    {
+      "epoch": 1.1985065339141256,
+      "grad_norm": 2.897993326187134,
+      "learning_rate": 3.3694649446494464e-06,
+      "loss": 1.4975,
+      "step": 963
+    },
+    {
+      "epoch": 1.1997510889856877,
+      "grad_norm": 2.739470958709717,
+      "learning_rate": 3.3671586715867163e-06,
+      "loss": 1.4202,
+      "step": 964
+    },
+    {
+      "epoch": 1.2009956440572496,
+      "grad_norm": 2.817786693572998,
+      "learning_rate": 3.3648523985239857e-06,
+      "loss": 1.5195,
+      "step": 965
+    },
+    {
+      "epoch": 1.2022401991288114,
+      "grad_norm": 2.8476386070251465,
+      "learning_rate": 3.362546125461255e-06,
+      "loss": 1.444,
+      "step": 966
+    },
+    {
+      "epoch": 1.2034847542003733,
+      "grad_norm": 2.804047107696533,
+      "learning_rate": 3.360239852398524e-06,
+      "loss": 1.4442,
+      "step": 967
+    },
+    {
+      "epoch": 1.2047293092719353,
+      "grad_norm": 2.9285430908203125,
+      "learning_rate": 3.3579335793357935e-06,
+      "loss": 1.4877,
+      "step": 968
+    },
+    {
+      "epoch": 1.2059738643434972,
+      "grad_norm": 2.8814878463745117,
+      "learning_rate": 3.3556273062730634e-06,
+      "loss": 1.5078,
+      "step": 969
+    },
+    {
+      "epoch": 1.207218419415059,
+      "grad_norm": 2.970728635787964,
+      "learning_rate": 3.3533210332103324e-06,
+      "loss": 1.4166,
+      "step": 970
+    },
+    {
+      "epoch": 1.207218419415059,
+      "eval_loss": 1.6635315418243408,
+      "eval_runtime": 47.2321,
+      "eval_samples_per_second": 21.172,
+      "eval_steps_per_second": 0.889,
+      "step": 970
+    },
+    {
+      "epoch": 1.208462974486621,
+      "grad_norm": 2.9836292266845703,
+      "learning_rate": 3.3510147601476018e-06,
+      "loss": 1.4777,
+      "step": 971
+    },
+    {
+      "epoch": 1.209707529558183,
+      "grad_norm": 2.8495216369628906,
+      "learning_rate": 3.348708487084871e-06,
+      "loss": 1.4829,
+      "step": 972
+    },
+    {
+      "epoch": 1.2109520846297448,
+      "grad_norm": 2.7709312438964844,
+      "learning_rate": 3.34640221402214e-06,
+      "loss": 1.4242,
+      "step": 973
+    },
+    {
+      "epoch": 1.2121966397013069,
+      "grad_norm": 2.999462366104126,
+      "learning_rate": 3.3440959409594096e-06,
+      "loss": 1.4148,
+      "step": 974
+    },
+    {
+      "epoch": 1.2134411947728687,
+      "grad_norm": 2.8719074726104736,
+      "learning_rate": 3.3417896678966795e-06,
+      "loss": 1.4196,
+      "step": 975
+    },
+    {
+      "epoch": 1.2146857498444306,
+      "grad_norm": 2.8469338417053223,
+      "learning_rate": 3.3394833948339485e-06,
+      "loss": 1.4941,
+      "step": 976
+    },
+    {
+      "epoch": 1.2159303049159926,
+      "grad_norm": 2.9780938625335693,
+      "learning_rate": 3.337177121771218e-06,
+      "loss": 1.513,
+      "step": 977
+    },
+    {
+      "epoch": 1.2171748599875545,
+      "grad_norm": 2.814760684967041,
+      "learning_rate": 3.3348708487084873e-06,
+      "loss": 1.4204,
+      "step": 978
+    },
+    {
+      "epoch": 1.2184194150591163,
+      "grad_norm": 3.016261339187622,
+      "learning_rate": 3.332564575645757e-06,
+      "loss": 1.4389,
+      "step": 979
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "grad_norm": 2.9385855197906494,
+      "learning_rate": 3.330258302583026e-06,
+      "loss": 1.4885,
+      "step": 980
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "eval_loss": 1.6688873767852783,
+      "eval_runtime": 46.2942,
+      "eval_samples_per_second": 21.601,
+      "eval_steps_per_second": 0.907,
+      "step": 980
+    },
+    {
+      "epoch": 1.2209085252022402,
+      "grad_norm": 3.1434757709503174,
+      "learning_rate": 3.3279520295202956e-06,
+      "loss": 1.5157,
+      "step": 981
+    },
+    {
+      "epoch": 1.222153080273802,
+      "grad_norm": 3.0104827880859375,
+      "learning_rate": 3.325645756457565e-06,
+      "loss": 1.49,
+      "step": 982
+    },
+    {
+      "epoch": 1.223397635345364,
+      "grad_norm": 3.1913645267486572,
+      "learning_rate": 3.323339483394834e-06,
+      "loss": 1.4937,
+      "step": 983
+    },
+    {
+      "epoch": 1.224642190416926,
+      "grad_norm": 3.19496750831604,
+      "learning_rate": 3.3210332103321034e-06,
+      "loss": 1.5197,
+      "step": 984
+    },
+    {
+      "epoch": 1.2258867454884879,
+      "grad_norm": 3.046232223510742,
+      "learning_rate": 3.3187269372693733e-06,
+      "loss": 1.4497,
+      "step": 985
+    },
+    {
+      "epoch": 1.2271313005600497,
+      "grad_norm": 2.850675344467163,
+      "learning_rate": 3.3164206642066423e-06,
+      "loss": 1.4595,
+      "step": 986
+    },
+    {
+      "epoch": 1.2283758556316118,
+      "grad_norm": 2.7744576930999756,
+      "learning_rate": 3.3141143911439117e-06,
+      "loss": 1.4726,
+      "step": 987
+    },
+    {
+      "epoch": 1.2296204107031736,
+      "grad_norm": 3.017153024673462,
+      "learning_rate": 3.311808118081181e-06,
+      "loss": 1.493,
+      "step": 988
+    },
+    {
+      "epoch": 1.2308649657747355,
+      "grad_norm": 2.8821120262145996,
+      "learning_rate": 3.3095018450184506e-06,
+      "loss": 1.4734,
+      "step": 989
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "grad_norm": 2.8805840015411377,
+      "learning_rate": 3.3071955719557196e-06,
+      "loss": 1.4562,
+      "step": 990
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "eval_loss": 1.669345736503601,
+      "eval_runtime": 47.0893,
+      "eval_samples_per_second": 21.236,
+      "eval_steps_per_second": 0.892,
+      "step": 990
+    },
+    {
+      "epoch": 1.2333540759178594,
+      "grad_norm": 2.7599008083343506,
+      "learning_rate": 3.3048892988929894e-06,
+      "loss": 1.4218,
+      "step": 991
+    },
+    {
+      "epoch": 1.2345986309894212,
+      "grad_norm": 2.9835240840911865,
+      "learning_rate": 3.302583025830259e-06,
+      "loss": 1.4596,
+      "step": 992
+    },
+    {
+      "epoch": 1.235843186060983,
+      "grad_norm": 2.843358039855957,
+      "learning_rate": 3.300276752767528e-06,
+      "loss": 1.4561,
+      "step": 993
+    },
+    {
+      "epoch": 1.2370877411325452,
+      "grad_norm": 2.7386093139648438,
+      "learning_rate": 3.2979704797047972e-06,
+      "loss": 1.4475,
+      "step": 994
+    },
+    {
+      "epoch": 1.238332296204107,
+      "grad_norm": 2.668161392211914,
+      "learning_rate": 3.2956642066420667e-06,
+      "loss": 1.4661,
+      "step": 995
+    },
+    {
+      "epoch": 1.2395768512756689,
+      "grad_norm": 2.947235107421875,
+      "learning_rate": 3.2933579335793357e-06,
+      "loss": 1.4358,
+      "step": 996
+    },
+    {
+      "epoch": 1.240821406347231,
+      "grad_norm": 2.8159878253936768,
+      "learning_rate": 3.2910516605166055e-06,
+      "loss": 1.4432,
+      "step": 997
+    },
+    {
+      "epoch": 1.2420659614187928,
+      "grad_norm": 2.896597385406494,
+      "learning_rate": 3.288745387453875e-06,
+      "loss": 1.5585,
+      "step": 998
+    },
+    {
+      "epoch": 1.2433105164903546,
+      "grad_norm": 2.80241322517395,
+      "learning_rate": 3.286439114391144e-06,
+      "loss": 1.5076,
+      "step": 999
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "grad_norm": 3.008939027786255,
+      "learning_rate": 3.2841328413284134e-06,
+      "loss": 1.4234,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "eval_loss": 1.6621588468551636,
+      "eval_runtime": 50.4926,
+      "eval_samples_per_second": 19.805,
+      "eval_steps_per_second": 0.832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2457996266334785,
+      "grad_norm": 2.9872381687164307,
+      "learning_rate": 3.281826568265683e-06,
+      "loss": 1.453,
+      "step": 1001
+    },
+    {
+      "epoch": 1.2470441817050404,
+      "grad_norm": 2.704676628112793,
+      "learning_rate": 3.2795202952029526e-06,
+      "loss": 1.4711,
+      "step": 1002
+    },
+    {
+      "epoch": 1.2482887367766025,
+      "grad_norm": 2.8100757598876953,
+      "learning_rate": 3.2772140221402216e-06,
+      "loss": 1.3884,
+      "step": 1003
+    },
+    {
+      "epoch": 1.2495332918481643,
+      "grad_norm": 2.902953624725342,
+      "learning_rate": 3.274907749077491e-06,
+      "loss": 1.4986,
+      "step": 1004
+    },
+    {
+      "epoch": 1.2507778469197262,
+      "grad_norm": 2.980163097381592,
+      "learning_rate": 3.2726014760147605e-06,
+      "loss": 1.5067,
+      "step": 1005
+    },
+    {
+      "epoch": 1.252022401991288,
+      "grad_norm": 2.8820345401763916,
+      "learning_rate": 3.2702952029520295e-06,
+      "loss": 1.4995,
+      "step": 1006
+    },
+    {
+      "epoch": 1.25326695706285,
+      "grad_norm": 2.8219635486602783,
+      "learning_rate": 3.2679889298892993e-06,
+      "loss": 1.4881,
+      "step": 1007
+    },
+    {
+      "epoch": 1.254511512134412,
+      "grad_norm": 2.8593199253082275,
+      "learning_rate": 3.2656826568265687e-06,
+      "loss": 1.4469,
+      "step": 1008
+    },
+    {
+      "epoch": 1.255756067205974,
+      "grad_norm": 2.888357639312744,
+      "learning_rate": 3.2633763837638377e-06,
+      "loss": 1.4869,
+      "step": 1009
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "grad_norm": 2.9985013008117676,
+      "learning_rate": 3.261070110701107e-06,
+      "loss": 1.4724,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "eval_loss": 1.6492141485214233,
+      "eval_runtime": 45.2197,
+      "eval_samples_per_second": 22.114,
+      "eval_steps_per_second": 0.929,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2582451773490977,
+      "grad_norm": 2.860780715942383,
+      "learning_rate": 3.2587638376383766e-06,
+      "loss": 1.4618,
+      "step": 1011
+    },
+    {
+      "epoch": 1.2594897324206595,
+      "grad_norm": 2.766479730606079,
+      "learning_rate": 3.2564575645756456e-06,
+      "loss": 1.4642,
+      "step": 1012
+    },
+    {
+      "epoch": 1.2607342874922216,
+      "grad_norm": 2.711759328842163,
+      "learning_rate": 3.2541512915129154e-06,
+      "loss": 1.4283,
+      "step": 1013
+    },
+    {
+      "epoch": 1.2619788425637835,
+      "grad_norm": 2.8365769386291504,
+      "learning_rate": 3.251845018450185e-06,
+      "loss": 1.4248,
+      "step": 1014
+    },
+    {
+      "epoch": 1.2632233976353453,
+      "grad_norm": 2.957620143890381,
+      "learning_rate": 3.2495387453874543e-06,
+      "loss": 1.4662,
+      "step": 1015
+    },
+    {
+      "epoch": 1.2644679527069074,
+      "grad_norm": 2.8066208362579346,
+      "learning_rate": 3.2472324723247233e-06,
+      "loss": 1.4447,
+      "step": 1016
+    },
+    {
+      "epoch": 1.2657125077784692,
+      "grad_norm": 2.902040958404541,
+      "learning_rate": 3.244926199261993e-06,
+      "loss": 1.4743,
+      "step": 1017
+    },
+    {
+      "epoch": 1.266957062850031,
+      "grad_norm": 2.7636687755584717,
+      "learning_rate": 3.2426199261992625e-06,
+      "loss": 1.4465,
+      "step": 1018
+    },
+    {
+      "epoch": 1.268201617921593,
+      "grad_norm": 2.95261549949646,
+      "learning_rate": 3.2403136531365315e-06,
+      "loss": 1.4769,
+      "step": 1019
+    },
+    {
+      "epoch": 1.269446172993155,
+      "grad_norm": 2.911128520965576,
+      "learning_rate": 3.238007380073801e-06,
+      "loss": 1.3843,
+      "step": 1020
+    },
+    {
+      "epoch": 1.269446172993155,
+      "eval_loss": 1.6526964902877808,
+      "eval_runtime": 48.2126,
+      "eval_samples_per_second": 20.741,
+      "eval_steps_per_second": 0.871,
+      "step": 1020
+    },
+    {
+      "epoch": 1.2706907280647168,
+      "grad_norm": 2.769951820373535,
+      "learning_rate": 3.2357011070110704e-06,
+      "loss": 1.4847,
+      "step": 1021
+    },
+    {
+      "epoch": 1.271935283136279,
+      "grad_norm": 3.0451674461364746,
+      "learning_rate": 3.2333948339483394e-06,
+      "loss": 1.4453,
+      "step": 1022
+    },
+    {
+      "epoch": 1.2731798382078408,
+      "grad_norm": 2.9543521404266357,
+      "learning_rate": 3.2310885608856092e-06,
+      "loss": 1.4789,
+      "step": 1023
+    },
+    {
+      "epoch": 1.2744243932794026,
+      "grad_norm": 2.9174394607543945,
+      "learning_rate": 3.2287822878228787e-06,
+      "loss": 1.4807,
+      "step": 1024
+    },
+    {
+      "epoch": 1.2756689483509644,
+      "grad_norm": 2.888258934020996,
+      "learning_rate": 3.2264760147601477e-06,
+      "loss": 1.4367,
+      "step": 1025
+    },
+    {
+      "epoch": 1.2769135034225265,
+      "grad_norm": 2.8346047401428223,
+      "learning_rate": 3.224169741697417e-06,
+      "loss": 1.4313,
+      "step": 1026
+    },
+    {
+      "epoch": 1.2781580584940884,
+      "grad_norm": 2.7531332969665527,
+      "learning_rate": 3.2218634686346865e-06,
+      "loss": 1.4018,
+      "step": 1027
+    },
+    {
+      "epoch": 1.2794026135656502,
+      "grad_norm": 3.136946439743042,
+      "learning_rate": 3.2195571955719564e-06,
+      "loss": 1.4417,
+      "step": 1028
+    },
+    {
+      "epoch": 1.2806471686372123,
+      "grad_norm": 2.9618165493011475,
+      "learning_rate": 3.2172509225092254e-06,
+      "loss": 1.5117,
+      "step": 1029
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "grad_norm": 3.0740747451782227,
+      "learning_rate": 3.2149446494464948e-06,
+      "loss": 1.4641,
+      "step": 1030
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "eval_loss": 1.6572761535644531,
+      "eval_runtime": 51.6696,
+      "eval_samples_per_second": 19.354,
+      "eval_steps_per_second": 0.813,
+      "step": 1030
+    },
+    {
+      "epoch": 1.283136278780336,
+      "grad_norm": 2.9225106239318848,
+      "learning_rate": 3.212638376383764e-06,
+      "loss": 1.4421,
+      "step": 1031
+    },
+    {
+      "epoch": 1.2843808338518978,
+      "grad_norm": 2.9530649185180664,
+      "learning_rate": 3.210332103321033e-06,
+      "loss": 1.5138,
+      "step": 1032
+    },
+    {
+      "epoch": 1.28562538892346,
+      "grad_norm": 2.8513238430023193,
+      "learning_rate": 3.208025830258303e-06,
+      "loss": 1.411,
+      "step": 1033
+    },
+    {
+      "epoch": 1.2868699439950217,
+      "grad_norm": 2.7440171241760254,
+      "learning_rate": 3.2057195571955725e-06,
+      "loss": 1.4182,
+      "step": 1034
+    },
+    {
+      "epoch": 1.2881144990665838,
+      "grad_norm": 2.8126630783081055,
+      "learning_rate": 3.2034132841328415e-06,
+      "loss": 1.4333,
+      "step": 1035
+    },
+    {
+      "epoch": 1.2893590541381457,
+      "grad_norm": 2.8598744869232178,
+      "learning_rate": 3.201107011070111e-06,
+      "loss": 1.439,
+      "step": 1036
+    },
+    {
+      "epoch": 1.2906036092097075,
+      "grad_norm": 2.901622772216797,
+      "learning_rate": 3.1988007380073803e-06,
+      "loss": 1.4461,
+      "step": 1037
+    },
+    {
+      "epoch": 1.2918481642812694,
+      "grad_norm": 2.7670180797576904,
+      "learning_rate": 3.1964944649446493e-06,
+      "loss": 1.4772,
+      "step": 1038
+    },
+    {
+      "epoch": 1.2930927193528314,
+      "grad_norm": 2.7860350608825684,
+      "learning_rate": 3.194188191881919e-06,
+      "loss": 1.4414,
+      "step": 1039
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "grad_norm": 2.822052478790283,
+      "learning_rate": 3.1918819188191886e-06,
+      "loss": 1.4192,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "eval_loss": 1.663284420967102,
+      "eval_runtime": 47.2822,
+      "eval_samples_per_second": 21.15,
+      "eval_steps_per_second": 0.888,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2955818294959551,
+      "grad_norm": 2.909109115600586,
+      "learning_rate": 3.189575645756458e-06,
+      "loss": 1.4795,
+      "step": 1041
+    },
+    {
+      "epoch": 1.2968263845675172,
+      "grad_norm": 2.933673143386841,
+      "learning_rate": 3.187269372693727e-06,
+      "loss": 1.5038,
+      "step": 1042
+    },
+    {
+      "epoch": 1.298070939639079,
+      "grad_norm": 3.032518148422241,
+      "learning_rate": 3.1849630996309964e-06,
+      "loss": 1.5133,
+      "step": 1043
+    },
+    {
+      "epoch": 1.299315494710641,
+      "grad_norm": 2.826040506362915,
+      "learning_rate": 3.1826568265682663e-06,
+      "loss": 1.4395,
+      "step": 1044
+    },
+    {
+      "epoch": 1.3005600497822027,
+      "grad_norm": 2.8152658939361572,
+      "learning_rate": 3.1803505535055353e-06,
+      "loss": 1.4787,
+      "step": 1045
+    },
+    {
+      "epoch": 1.3018046048537648,
+      "grad_norm": 2.9410431385040283,
+      "learning_rate": 3.1780442804428047e-06,
+      "loss": 1.4375,
+      "step": 1046
+    },
+    {
+      "epoch": 1.3030491599253267,
+      "grad_norm": 2.948636531829834,
+      "learning_rate": 3.175738007380074e-06,
+      "loss": 1.4847,
+      "step": 1047
+    },
+    {
+      "epoch": 1.3042937149968887,
+      "grad_norm": 2.6287615299224854,
+      "learning_rate": 3.173431734317343e-06,
+      "loss": 1.4186,
+      "step": 1048
+    },
+    {
+      "epoch": 1.3055382700684506,
+      "grad_norm": 2.836029052734375,
+      "learning_rate": 3.1711254612546125e-06,
+      "loss": 1.4251,
+      "step": 1049
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "grad_norm": 2.982081651687622,
+      "learning_rate": 3.1688191881918824e-06,
+      "loss": 1.4951,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "eval_loss": 1.6578067541122437,
+      "eval_runtime": 47.5847,
+      "eval_samples_per_second": 21.015,
+      "eval_steps_per_second": 0.883,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3080273802115743,
+      "grad_norm": 2.9297118186950684,
+      "learning_rate": 3.166512915129152e-06,
+      "loss": 1.4699,
+      "step": 1051
+    },
+    {
+      "epoch": 1.3092719352831363,
+      "grad_norm": 2.906435251235962,
+      "learning_rate": 3.164206642066421e-06,
+      "loss": 1.4562,
+      "step": 1052
+    },
+    {
+      "epoch": 1.3105164903546982,
+      "grad_norm": 2.8655896186828613,
+      "learning_rate": 3.1619003690036902e-06,
+      "loss": 1.4351,
+      "step": 1053
+    },
+    {
+      "epoch": 1.31176104542626,
+      "grad_norm": 2.779244899749756,
+      "learning_rate": 3.15959409594096e-06,
+      "loss": 1.4479,
+      "step": 1054
+    },
+    {
+      "epoch": 1.3130056004978221,
+      "grad_norm": 2.9863739013671875,
+      "learning_rate": 3.157287822878229e-06,
+      "loss": 1.4964,
+      "step": 1055
+    },
+    {
+      "epoch": 1.314250155569384,
+      "grad_norm": 2.821871042251587,
+      "learning_rate": 3.1549815498154985e-06,
+      "loss": 1.4173,
+      "step": 1056
+    },
+    {
+      "epoch": 1.3154947106409458,
+      "grad_norm": 2.8573882579803467,
+      "learning_rate": 3.152675276752768e-06,
+      "loss": 1.4877,
+      "step": 1057
+    },
+    {
+      "epoch": 1.3167392657125077,
+      "grad_norm": 2.9824137687683105,
+      "learning_rate": 3.150369003690037e-06,
+      "loss": 1.5124,
+      "step": 1058
+    },
+    {
+      "epoch": 1.3179838207840697,
+      "grad_norm": 2.881098747253418,
+      "learning_rate": 3.1480627306273063e-06,
+      "loss": 1.4346,
+      "step": 1059
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "grad_norm": 2.918782949447632,
+      "learning_rate": 3.145756457564576e-06,
+      "loss": 1.4797,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "eval_loss": 1.6510179042816162,
+      "eval_runtime": 53.6356,
+      "eval_samples_per_second": 18.644,
+      "eval_steps_per_second": 0.783,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3204729309271936,
+      "grad_norm": 3.045776128768921,
+      "learning_rate": 3.143450184501845e-06,
+      "loss": 1.5039,
+      "step": 1061
+    },
+    {
+      "epoch": 1.3217174859987555,
+      "grad_norm": 3.041501760482788,
+      "learning_rate": 3.1411439114391146e-06,
+      "loss": 1.4878,
+      "step": 1062
+    },
+    {
+      "epoch": 1.3229620410703173,
+      "grad_norm": 2.917837142944336,
+      "learning_rate": 3.138837638376384e-06,
+      "loss": 1.4202,
+      "step": 1063
+    },
+    {
+      "epoch": 1.3242065961418792,
+      "grad_norm": 2.708070755004883,
+      "learning_rate": 3.1365313653136535e-06,
+      "loss": 1.4071,
+      "step": 1064
+    },
+    {
+      "epoch": 1.3254511512134413,
+      "grad_norm": 2.9146156311035156,
+      "learning_rate": 3.1342250922509225e-06,
+      "loss": 1.4428,
+      "step": 1065
+    },
+    {
+      "epoch": 1.326695706285003,
+      "grad_norm": 3.220919132232666,
+      "learning_rate": 3.1319188191881923e-06,
+      "loss": 1.4733,
+      "step": 1066
+    },
+    {
+      "epoch": 1.327940261356565,
+      "grad_norm": 2.9127554893493652,
+      "learning_rate": 3.1296125461254617e-06,
+      "loss": 1.4554,
+      "step": 1067
+    },
+    {
+      "epoch": 1.329184816428127,
+      "grad_norm": 3.053191661834717,
+      "learning_rate": 3.1273062730627307e-06,
+      "loss": 1.4044,
+      "step": 1068
+    },
+    {
+      "epoch": 1.3304293714996889,
+      "grad_norm": 3.2937021255493164,
+      "learning_rate": 3.125e-06,
+      "loss": 1.4949,
+      "step": 1069
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "grad_norm": 2.976717948913574,
+      "learning_rate": 3.12269372693727e-06,
+      "loss": 1.446,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "eval_loss": 1.6503105163574219,
+      "eval_runtime": 50.7579,
+      "eval_samples_per_second": 19.701,
+      "eval_steps_per_second": 0.827,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3329184816428126,
+      "grad_norm": 3.0860936641693115,
+      "learning_rate": 3.120387453874539e-06,
+      "loss": 1.4055,
+      "step": 1071
+    },
+    {
+      "epoch": 1.3341630367143746,
+      "grad_norm": 3.0401217937469482,
+      "learning_rate": 3.1180811808118084e-06,
+      "loss": 1.4688,
+      "step": 1072
+    },
+    {
+      "epoch": 1.3354075917859365,
+      "grad_norm": 2.953253984451294,
+      "learning_rate": 3.115774907749078e-06,
+      "loss": 1.5001,
+      "step": 1073
+    },
+    {
+      "epoch": 1.3366521468574986,
+      "grad_norm": 3.0665059089660645,
+      "learning_rate": 3.113468634686347e-06,
+      "loss": 1.4217,
+      "step": 1074
+    },
+    {
+      "epoch": 1.3378967019290604,
+      "grad_norm": 3.121600389480591,
+      "learning_rate": 3.1111623616236163e-06,
+      "loss": 1.4508,
+      "step": 1075
+    },
+    {
+      "epoch": 1.3391412570006223,
+      "grad_norm": 2.8057339191436768,
+      "learning_rate": 3.108856088560886e-06,
+      "loss": 1.4241,
+      "step": 1076
+    },
+    {
+      "epoch": 1.340385812072184,
+      "grad_norm": 3.0147411823272705,
+      "learning_rate": 3.1065498154981555e-06,
+      "loss": 1.479,
+      "step": 1077
+    },
+    {
+      "epoch": 1.3416303671437462,
+      "grad_norm": 3.030010223388672,
+      "learning_rate": 3.1042435424354245e-06,
+      "loss": 1.4861,
+      "step": 1078
+    },
+    {
+      "epoch": 1.342874922215308,
+      "grad_norm": 3.027998208999634,
+      "learning_rate": 3.101937269372694e-06,
+      "loss": 1.4916,
+      "step": 1079
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "grad_norm": 2.9252536296844482,
+      "learning_rate": 3.0996309963099634e-06,
+      "loss": 1.418,
+      "step": 1080
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "eval_loss": 1.6490333080291748,
+      "eval_runtime": 47.4026,
+      "eval_samples_per_second": 21.096,
+      "eval_steps_per_second": 0.886,
+      "step": 1080
+    },
+    {
+      "epoch": 1.345364032358432,
+      "grad_norm": 3.110847234725952,
+      "learning_rate": 3.0973247232472324e-06,
+      "loss": 1.4127,
+      "step": 1081
+    },
+    {
+      "epoch": 1.3466085874299938,
+      "grad_norm": 2.846491575241089,
+      "learning_rate": 3.0950184501845022e-06,
+      "loss": 1.444,
+      "step": 1082
+    },
+    {
+      "epoch": 1.3478531425015556,
+      "grad_norm": 2.950533151626587,
+      "learning_rate": 3.0927121771217716e-06,
+      "loss": 1.4456,
+      "step": 1083
+    },
+    {
+      "epoch": 1.3490976975731175,
+      "grad_norm": 2.875964403152466,
+      "learning_rate": 3.0904059040590406e-06,
+      "loss": 1.4485,
+      "step": 1084
+    },
+    {
+      "epoch": 1.3503422526446796,
+      "grad_norm": 2.9612326622009277,
+      "learning_rate": 3.08809963099631e-06,
+      "loss": 1.4446,
+      "step": 1085
+    },
+    {
+      "epoch": 1.3515868077162414,
+      "grad_norm": 2.925297975540161,
+      "learning_rate": 3.0857933579335795e-06,
+      "loss": 1.4847,
+      "step": 1086
+    },
+    {
+      "epoch": 1.3528313627878035,
+      "grad_norm": 2.991616725921631,
+      "learning_rate": 3.0834870848708485e-06,
+      "loss": 1.5151,
+      "step": 1087
+    },
+    {
+      "epoch": 1.3540759178593653,
+      "grad_norm": 2.96329927444458,
+      "learning_rate": 3.0811808118081183e-06,
+      "loss": 1.4574,
+      "step": 1088
+    },
+    {
+      "epoch": 1.3553204729309272,
+      "grad_norm": 2.948702573776245,
+      "learning_rate": 3.0788745387453878e-06,
+      "loss": 1.4585,
+      "step": 1089
+    },
+    {
+      "epoch": 1.356565028002489,
+      "grad_norm": 3.218463659286499,
+      "learning_rate": 3.076568265682657e-06,
+      "loss": 1.4439,
+      "step": 1090
+    },
+    {
+      "epoch": 1.356565028002489,
+      "eval_loss": 1.645838975906372,
+      "eval_runtime": 47.048,
+      "eval_samples_per_second": 21.255,
+      "eval_steps_per_second": 0.893,
+      "step": 1090
+    },
+    {
+      "epoch": 1.357809583074051,
+      "grad_norm": 2.9799578189849854,
+      "learning_rate": 3.074261992619926e-06,
+      "loss": 1.4883,
+      "step": 1091
+    },
+    {
+      "epoch": 1.359054138145613,
+      "grad_norm": 2.7660505771636963,
+      "learning_rate": 3.071955719557196e-06,
+      "loss": 1.4349,
+      "step": 1092
+    },
+    {
+      "epoch": 1.3602986932171748,
+      "grad_norm": 3.0201916694641113,
+      "learning_rate": 3.0696494464944655e-06,
+      "loss": 1.418,
+      "step": 1093
+    },
+    {
+      "epoch": 1.3615432482887369,
+      "grad_norm": 2.844207763671875,
+      "learning_rate": 3.0673431734317345e-06,
+      "loss": 1.4769,
+      "step": 1094
+    },
+    {
+      "epoch": 1.3627878033602987,
+      "grad_norm": 2.8306753635406494,
+      "learning_rate": 3.065036900369004e-06,
+      "loss": 1.4653,
+      "step": 1095
+    },
+    {
+      "epoch": 1.3640323584318605,
+      "grad_norm": 2.805023431777954,
+      "learning_rate": 3.0627306273062733e-06,
+      "loss": 1.4771,
+      "step": 1096
+    },
+    {
+      "epoch": 1.3652769135034224,
+      "grad_norm": 3.020883083343506,
+      "learning_rate": 3.0604243542435423e-06,
+      "loss": 1.4237,
+      "step": 1097
+    },
+    {
+      "epoch": 1.3665214685749845,
+      "grad_norm": 2.8395731449127197,
+      "learning_rate": 3.058118081180812e-06,
+      "loss": 1.4647,
+      "step": 1098
+    },
+    {
+      "epoch": 1.3677660236465463,
+      "grad_norm": 2.871156692504883,
+      "learning_rate": 3.0558118081180816e-06,
+      "loss": 1.4196,
+      "step": 1099
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "grad_norm": 2.974905014038086,
+      "learning_rate": 3.053505535055351e-06,
+      "loss": 1.4779,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "eval_loss": 1.651503562927246,
+      "eval_runtime": 54.2713,
+      "eval_samples_per_second": 18.426,
+      "eval_steps_per_second": 0.774,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3702551337896702,
+      "grad_norm": 2.7591328620910645,
+      "learning_rate": 3.05119926199262e-06,
+      "loss": 1.3899,
+      "step": 1101
+    },
+    {
+      "epoch": 1.371499688861232,
+      "grad_norm": 2.8575668334960938,
+      "learning_rate": 3.0488929889298894e-06,
+      "loss": 1.4482,
+      "step": 1102
+    },
+    {
+      "epoch": 1.372744243932794,
+      "grad_norm": 2.9049339294433594,
+      "learning_rate": 3.0465867158671593e-06,
+      "loss": 1.4785,
+      "step": 1103
+    },
+    {
+      "epoch": 1.373988799004356,
+      "grad_norm": 2.816453695297241,
+      "learning_rate": 3.0442804428044283e-06,
+      "loss": 1.4601,
+      "step": 1104
+    },
+    {
+      "epoch": 1.3752333540759178,
+      "grad_norm": 2.8844189643859863,
+      "learning_rate": 3.0419741697416977e-06,
+      "loss": 1.496,
+      "step": 1105
+    },
+    {
+      "epoch": 1.3764779091474797,
+      "grad_norm": 3.014744281768799,
+      "learning_rate": 3.039667896678967e-06,
+      "loss": 1.4098,
+      "step": 1106
+    },
+    {
+      "epoch": 1.3777224642190418,
+      "grad_norm": 2.8243560791015625,
+      "learning_rate": 3.037361623616236e-06,
+      "loss": 1.4086,
+      "step": 1107
+    },
+    {
+      "epoch": 1.3789670192906036,
+      "grad_norm": 2.8333253860473633,
+      "learning_rate": 3.035055350553506e-06,
+      "loss": 1.3516,
+      "step": 1108
+    },
+    {
+      "epoch": 1.3802115743621655,
+      "grad_norm": 2.926455020904541,
+      "learning_rate": 3.0327490774907754e-06,
+      "loss": 1.3976,
+      "step": 1109
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "grad_norm": 2.900937080383301,
+      "learning_rate": 3.0304428044280444e-06,
+      "loss": 1.4336,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "eval_loss": 1.647048830986023,
+      "eval_runtime": 42.3955,
+      "eval_samples_per_second": 23.587,
+      "eval_steps_per_second": 0.991,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3827006845052894,
+      "grad_norm": 3.0756969451904297,
+      "learning_rate": 3.028136531365314e-06,
+      "loss": 1.4674,
+      "step": 1111
+    },
+    {
+      "epoch": 1.3839452395768512,
+      "grad_norm": 2.7754578590393066,
+      "learning_rate": 3.0258302583025832e-06,
+      "loss": 1.4435,
+      "step": 1112
+    },
+    {
+      "epoch": 1.3851897946484133,
+      "grad_norm": 2.8011393547058105,
+      "learning_rate": 3.023523985239853e-06,
+      "loss": 1.4212,
+      "step": 1113
+    },
+    {
+      "epoch": 1.3864343497199751,
+      "grad_norm": 2.7063710689544678,
+      "learning_rate": 3.021217712177122e-06,
+      "loss": 1.3769,
+      "step": 1114
+    },
+    {
+      "epoch": 1.387678904791537,
+      "grad_norm": 2.830411434173584,
+      "learning_rate": 3.0189114391143915e-06,
+      "loss": 1.4832,
+      "step": 1115
+    },
+    {
+      "epoch": 1.3889234598630988,
+      "grad_norm": 2.9075911045074463,
+      "learning_rate": 3.016605166051661e-06,
+      "loss": 1.486,
+      "step": 1116
+    },
+    {
+      "epoch": 1.390168014934661,
+      "grad_norm": 2.847822666168213,
+      "learning_rate": 3.01429889298893e-06,
+      "loss": 1.424,
+      "step": 1117
+    },
+    {
+      "epoch": 1.3914125700062228,
+      "grad_norm": 2.8822033405303955,
+      "learning_rate": 3.0119926199261993e-06,
+      "loss": 1.4218,
+      "step": 1118
+    },
+    {
+      "epoch": 1.3926571250777848,
+      "grad_norm": 2.8944740295410156,
+      "learning_rate": 3.009686346863469e-06,
+      "loss": 1.4283,
+      "step": 1119
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "grad_norm": 2.765190839767456,
+      "learning_rate": 3.007380073800738e-06,
+      "loss": 1.4622,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "eval_loss": 1.6451724767684937,
+      "eval_runtime": 43.7449,
+      "eval_samples_per_second": 22.86,
+      "eval_steps_per_second": 0.96,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3951462352209085,
+      "grad_norm": 2.877002716064453,
+      "learning_rate": 3.0050738007380076e-06,
+      "loss": 1.4145,
+      "step": 1121
+    },
+    {
+      "epoch": 1.3963907902924704,
+      "grad_norm": 2.870389699935913,
+      "learning_rate": 3.002767527675277e-06,
+      "loss": 1.4401,
+      "step": 1122
+    },
+    {
+      "epoch": 1.3976353453640322,
+      "grad_norm": 2.7788920402526855,
+      "learning_rate": 3.000461254612546e-06,
+      "loss": 1.4514,
+      "step": 1123
+    },
+    {
+      "epoch": 1.3988799004355943,
+      "grad_norm": 2.9269332885742188,
+      "learning_rate": 2.998154981549816e-06,
+      "loss": 1.3961,
+      "step": 1124
+    },
+    {
+      "epoch": 1.4001244555071561,
+      "grad_norm": 3.013291835784912,
+      "learning_rate": 2.9958487084870853e-06,
+      "loss": 1.4597,
+      "step": 1125
+    },
+    {
+      "epoch": 1.4013690105787182,
+      "grad_norm": 2.9063880443573,
+      "learning_rate": 2.9935424354243547e-06,
+      "loss": 1.4668,
+      "step": 1126
+    },
+    {
+      "epoch": 1.40261356565028,
+      "grad_norm": 2.8926970958709717,
+      "learning_rate": 2.9912361623616237e-06,
+      "loss": 1.4332,
+      "step": 1127
+    },
+    {
+      "epoch": 1.403858120721842,
+      "grad_norm": 2.8563647270202637,
+      "learning_rate": 2.988929889298893e-06,
+      "loss": 1.407,
+      "step": 1128
+    },
+    {
+      "epoch": 1.4051026757934038,
+      "grad_norm": 2.8725359439849854,
+      "learning_rate": 2.986623616236163e-06,
+      "loss": 1.4434,
+      "step": 1129
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "grad_norm": 2.906898260116577,
+      "learning_rate": 2.984317343173432e-06,
+      "loss": 1.4745,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "eval_loss": 1.647884726524353,
+      "eval_runtime": 44.9221,
+      "eval_samples_per_second": 22.261,
+      "eval_steps_per_second": 0.935,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4075917859365277,
+      "grad_norm": 2.8297719955444336,
+      "learning_rate": 2.9820110701107014e-06,
+      "loss": 1.4542,
+      "step": 1131
+    },
+    {
+      "epoch": 1.4088363410080897,
+      "grad_norm": 2.841031551361084,
+      "learning_rate": 2.979704797047971e-06,
+      "loss": 1.4105,
+      "step": 1132
+    },
+    {
+      "epoch": 1.4100808960796516,
+      "grad_norm": 2.867791175842285,
+      "learning_rate": 2.97739852398524e-06,
+      "loss": 1.4409,
+      "step": 1133
+    },
+    {
+      "epoch": 1.4113254511512134,
+      "grad_norm": 2.8156039714813232,
+      "learning_rate": 2.9750922509225093e-06,
+      "loss": 1.4284,
+      "step": 1134
+    },
+    {
+      "epoch": 1.4125700062227753,
+      "grad_norm": 2.825985908508301,
+      "learning_rate": 2.972785977859779e-06,
+      "loss": 1.4594,
+      "step": 1135
+    },
+    {
+      "epoch": 1.4138145612943374,
+      "grad_norm": 3.0134711265563965,
+      "learning_rate": 2.970479704797048e-06,
+      "loss": 1.4743,
+      "step": 1136
+    },
+    {
+      "epoch": 1.4150591163658992,
+      "grad_norm": 2.9829142093658447,
+      "learning_rate": 2.9681734317343175e-06,
+      "loss": 1.4935,
+      "step": 1137
+    },
+    {
+      "epoch": 1.416303671437461,
+      "grad_norm": 3.067620277404785,
+      "learning_rate": 2.965867158671587e-06,
+      "loss": 1.4266,
+      "step": 1138
+    },
+    {
+      "epoch": 1.4175482265090231,
+      "grad_norm": 2.8969919681549072,
+      "learning_rate": 2.9635608856088564e-06,
+      "loss": 1.451,
+      "step": 1139
+    },
+    {
+      "epoch": 1.418792781580585,
+      "grad_norm": 2.8706412315368652,
+      "learning_rate": 2.9612546125461254e-06,
+      "loss": 1.4834,
+      "step": 1140
+    },
+    {
+      "epoch": 1.418792781580585,
+      "eval_loss": 1.643480658531189,
+      "eval_runtime": 44.743,
+      "eval_samples_per_second": 22.35,
+      "eval_steps_per_second": 0.939,
+      "step": 1140
+    },
+    {
+      "epoch": 1.4200373366521468,
+      "grad_norm": 2.8801848888397217,
+      "learning_rate": 2.958948339483395e-06,
+      "loss": 1.4258,
+      "step": 1141
+    },
+    {
+      "epoch": 1.4212818917237087,
+      "grad_norm": 2.774635076522827,
+      "learning_rate": 2.9566420664206646e-06,
+      "loss": 1.416,
+      "step": 1142
+    },
+    {
+      "epoch": 1.4225264467952707,
+      "grad_norm": 2.8460140228271484,
+      "learning_rate": 2.9543357933579336e-06,
+      "loss": 1.4213,
+      "step": 1143
+    },
+    {
+      "epoch": 1.4237710018668326,
+      "grad_norm": 2.907888889312744,
+      "learning_rate": 2.952029520295203e-06,
+      "loss": 1.4507,
+      "step": 1144
+    },
+    {
+      "epoch": 1.4250155569383947,
+      "grad_norm": 2.789041757583618,
+      "learning_rate": 2.949723247232473e-06,
+      "loss": 1.4257,
+      "step": 1145
+    },
+    {
+      "epoch": 1.4262601120099565,
+      "grad_norm": 2.78859806060791,
+      "learning_rate": 2.947416974169742e-06,
+      "loss": 1.4377,
+      "step": 1146
+    },
+    {
+      "epoch": 1.4275046670815184,
+      "grad_norm": 2.840303421020508,
+      "learning_rate": 2.9451107011070113e-06,
+      "loss": 1.4684,
+      "step": 1147
+    },
+    {
+      "epoch": 1.4287492221530802,
+      "grad_norm": 2.8800530433654785,
+      "learning_rate": 2.9428044280442807e-06,
+      "loss": 1.4912,
+      "step": 1148
+    },
+    {
+      "epoch": 1.4299937772246423,
+      "grad_norm": 2.841092824935913,
+      "learning_rate": 2.9404981549815497e-06,
+      "loss": 1.4234,
+      "step": 1149
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "grad_norm": 2.8226537704467773,
+      "learning_rate": 2.938191881918819e-06,
+      "loss": 1.4578,
+      "step": 1150
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "eval_loss": 1.643249273300171,
+      "eval_runtime": 43.5586,
+      "eval_samples_per_second": 22.958,
+      "eval_steps_per_second": 0.964,
+      "step": 1150
+    },
+    {
+      "epoch": 1.432482887367766,
+      "grad_norm": 2.869736909866333,
+      "learning_rate": 2.935885608856089e-06,
+      "loss": 1.4701,
+      "step": 1151
+    },
+    {
+      "epoch": 1.433727442439328,
+      "grad_norm": 2.689211130142212,
+      "learning_rate": 2.9335793357933584e-06,
+      "loss": 1.4483,
+      "step": 1152
+    },
+    {
+      "epoch": 1.4349719975108899,
+      "grad_norm": 2.9267077445983887,
+      "learning_rate": 2.9312730627306274e-06,
+      "loss": 1.4446,
+      "step": 1153
+    },
+    {
+      "epoch": 1.4362165525824517,
+      "grad_norm": 2.906862497329712,
+      "learning_rate": 2.928966789667897e-06,
+      "loss": 1.4524,
+      "step": 1154
+    },
+    {
+      "epoch": 1.4374611076540136,
+      "grad_norm": 2.8249800205230713,
+      "learning_rate": 2.9266605166051663e-06,
+      "loss": 1.4208,
+      "step": 1155
+    },
+    {
+      "epoch": 1.4387056627255757,
+      "grad_norm": 2.9836032390594482,
+      "learning_rate": 2.9243542435424353e-06,
+      "loss": 1.4581,
+      "step": 1156
+    },
+    {
+      "epoch": 1.4399502177971375,
+      "grad_norm": 2.9595284461975098,
+      "learning_rate": 2.922047970479705e-06,
+      "loss": 1.4141,
+      "step": 1157
+    },
+    {
+      "epoch": 1.4411947728686996,
+      "grad_norm": 3.0363423824310303,
+      "learning_rate": 2.9197416974169746e-06,
+      "loss": 1.4859,
+      "step": 1158
+    },
+    {
+      "epoch": 1.4424393279402614,
+      "grad_norm": 2.911201000213623,
+      "learning_rate": 2.9174354243542436e-06,
+      "loss": 1.4346,
+      "step": 1159
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "grad_norm": 2.8617193698883057,
+      "learning_rate": 2.915129151291513e-06,
+      "loss": 1.433,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "eval_loss": 1.6437232494354248,
+      "eval_runtime": 43.961,
+      "eval_samples_per_second": 22.747,
+      "eval_steps_per_second": 0.955,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4449284380833851,
+      "grad_norm": 2.9584295749664307,
+      "learning_rate": 2.912822878228783e-06,
+      "loss": 1.4847,
+      "step": 1161
+    },
+    {
+      "epoch": 1.4461729931549472,
+      "grad_norm": 2.8209400177001953,
+      "learning_rate": 2.9105166051660522e-06,
+      "loss": 1.421,
+      "step": 1162
+    },
+    {
+      "epoch": 1.447417548226509,
+      "grad_norm": 2.847637414932251,
+      "learning_rate": 2.9082103321033212e-06,
+      "loss": 1.4326,
+      "step": 1163
+    },
+    {
+      "epoch": 1.4486621032980709,
+      "grad_norm": 2.836228132247925,
+      "learning_rate": 2.9059040590405907e-06,
+      "loss": 1.4459,
+      "step": 1164
+    },
+    {
+      "epoch": 1.449906658369633,
+      "grad_norm": 2.8327207565307617,
+      "learning_rate": 2.90359778597786e-06,
+      "loss": 1.4189,
+      "step": 1165
+    },
+    {
+      "epoch": 1.4511512134411948,
+      "grad_norm": 2.880643606185913,
+      "learning_rate": 2.901291512915129e-06,
+      "loss": 1.4214,
+      "step": 1166
+    },
+    {
+      "epoch": 1.4523957685127566,
+      "grad_norm": 2.9637348651885986,
+      "learning_rate": 2.898985239852399e-06,
+      "loss": 1.4471,
+      "step": 1167
+    },
+    {
+      "epoch": 1.4536403235843185,
+      "grad_norm": 3.006145477294922,
+      "learning_rate": 2.8966789667896684e-06,
+      "loss": 1.4228,
+      "step": 1168
+    },
+    {
+      "epoch": 1.4548848786558806,
+      "grad_norm": 2.8547627925872803,
+      "learning_rate": 2.8943726937269374e-06,
+      "loss": 1.4707,
+      "step": 1169
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "grad_norm": 2.973092794418335,
+      "learning_rate": 2.8920664206642068e-06,
+      "loss": 1.4804,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "eval_loss": 1.6340677738189697,
+      "eval_runtime": 44.6094,
+      "eval_samples_per_second": 22.417,
+      "eval_steps_per_second": 0.942,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4573739887990045,
+      "grad_norm": 2.9433603286743164,
+      "learning_rate": 2.889760147601476e-06,
+      "loss": 1.4354,
+      "step": 1171
+    },
+    {
+      "epoch": 1.4586185438705663,
+      "grad_norm": 2.872236728668213,
+      "learning_rate": 2.887453874538745e-06,
+      "loss": 1.418,
+      "step": 1172
+    },
+    {
+      "epoch": 1.4598630989421282,
+      "grad_norm": 2.847557783126831,
+      "learning_rate": 2.885147601476015e-06,
+      "loss": 1.4306,
+      "step": 1173
+    },
+    {
+      "epoch": 1.46110765401369,
+      "grad_norm": 2.7554969787597656,
+      "learning_rate": 2.8828413284132845e-06,
+      "loss": 1.4295,
+      "step": 1174
+    },
+    {
+      "epoch": 1.462352209085252,
+      "grad_norm": 2.8754842281341553,
+      "learning_rate": 2.880535055350554e-06,
+      "loss": 1.4482,
+      "step": 1175
+    },
+    {
+      "epoch": 1.463596764156814,
+      "grad_norm": 3.0115292072296143,
+      "learning_rate": 2.878228782287823e-06,
+      "loss": 1.4428,
+      "step": 1176
+    },
+    {
+      "epoch": 1.4648413192283758,
+      "grad_norm": 2.8976168632507324,
+      "learning_rate": 2.8759225092250923e-06,
+      "loss": 1.4707,
+      "step": 1177
+    },
+    {
+      "epoch": 1.4660858742999379,
+      "grad_norm": 2.7981812953948975,
+      "learning_rate": 2.873616236162362e-06,
+      "loss": 1.4693,
+      "step": 1178
+    },
+    {
+      "epoch": 1.4673304293714997,
+      "grad_norm": 2.8562371730804443,
+      "learning_rate": 2.871309963099631e-06,
+      "loss": 1.4242,
+      "step": 1179
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "grad_norm": 2.8705570697784424,
+      "learning_rate": 2.8690036900369006e-06,
+      "loss": 1.4725,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "eval_loss": 1.6338286399841309,
+      "eval_runtime": 44.7171,
+      "eval_samples_per_second": 22.363,
+      "eval_steps_per_second": 0.939,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4698195395146234,
+      "grad_norm": 2.8025710582733154,
+      "learning_rate": 2.86669741697417e-06,
+      "loss": 1.427,
+      "step": 1181
+    },
+    {
+      "epoch": 1.4710640945861855,
+      "grad_norm": 2.845151901245117,
+      "learning_rate": 2.864391143911439e-06,
+      "loss": 1.4402,
+      "step": 1182
+    },
+    {
+      "epoch": 1.4723086496577473,
+      "grad_norm": 2.943373680114746,
+      "learning_rate": 2.862084870848709e-06,
+      "loss": 1.5194,
+      "step": 1183
+    },
+    {
+      "epoch": 1.4735532047293094,
+      "grad_norm": 2.896955966949463,
+      "learning_rate": 2.8597785977859783e-06,
+      "loss": 1.4242,
+      "step": 1184
+    },
+    {
+      "epoch": 1.4747977598008712,
+      "grad_norm": 3.131305456161499,
+      "learning_rate": 2.8574723247232473e-06,
+      "loss": 1.5008,
+      "step": 1185
+    },
+    {
+      "epoch": 1.476042314872433,
+      "grad_norm": 2.8605735301971436,
+      "learning_rate": 2.8551660516605167e-06,
+      "loss": 1.3533,
+      "step": 1186
+    },
+    {
+      "epoch": 1.477286869943995,
+      "grad_norm": 2.9158811569213867,
+      "learning_rate": 2.852859778597786e-06,
+      "loss": 1.4404,
+      "step": 1187
+    },
+    {
+      "epoch": 1.478531425015557,
+      "grad_norm": 2.7795321941375732,
+      "learning_rate": 2.850553505535056e-06,
+      "loss": 1.3977,
+      "step": 1188
+    },
+    {
+      "epoch": 1.4797759800871189,
+      "grad_norm": 2.8194212913513184,
+      "learning_rate": 2.848247232472325e-06,
+      "loss": 1.4111,
+      "step": 1189
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "grad_norm": 2.983750104904175,
+      "learning_rate": 2.8459409594095944e-06,
+      "loss": 1.4061,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "eval_loss": 1.6375409364700317,
+      "eval_runtime": 45.4026,
+      "eval_samples_per_second": 22.025,
+      "eval_steps_per_second": 0.925,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4822650902302428,
+      "grad_norm": 3.0246665477752686,
+      "learning_rate": 2.843634686346864e-06,
+      "loss": 1.4532,
+      "step": 1191
+    },
+    {
+      "epoch": 1.4835096453018046,
+      "grad_norm": 2.8856449127197266,
+      "learning_rate": 2.841328413284133e-06,
+      "loss": 1.4528,
+      "step": 1192
+    },
+    {
+      "epoch": 1.4847542003733665,
+      "grad_norm": 2.848987102508545,
+      "learning_rate": 2.8390221402214022e-06,
+      "loss": 1.4369,
+      "step": 1193
+    },
+    {
+      "epoch": 1.4859987554449283,
+      "grad_norm": 2.970419406890869,
+      "learning_rate": 2.836715867158672e-06,
+      "loss": 1.4605,
+      "step": 1194
+    },
+    {
+      "epoch": 1.4872433105164904,
+      "grad_norm": 3.156613826751709,
+      "learning_rate": 2.834409594095941e-06,
+      "loss": 1.4489,
+      "step": 1195
+    },
+    {
+      "epoch": 1.4884878655880522,
+      "grad_norm": 3.095349073410034,
+      "learning_rate": 2.8321033210332105e-06,
+      "loss": 1.4604,
+      "step": 1196
+    },
+    {
+      "epoch": 1.4897324206596143,
+      "grad_norm": 2.870742082595825,
+      "learning_rate": 2.82979704797048e-06,
+      "loss": 1.4284,
+      "step": 1197
+    },
+    {
+      "epoch": 1.4909769757311762,
+      "grad_norm": 2.9415056705474854,
+      "learning_rate": 2.827490774907749e-06,
+      "loss": 1.4715,
+      "step": 1198
+    },
+    {
+      "epoch": 1.492221530802738,
+      "grad_norm": 2.8297767639160156,
+      "learning_rate": 2.8251845018450188e-06,
+      "loss": 1.4508,
+      "step": 1199
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "grad_norm": 2.872086524963379,
+      "learning_rate": 2.822878228782288e-06,
+      "loss": 1.5098,
+      "step": 1200
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "eval_loss": 1.6357526779174805,
+      "eval_runtime": 51.422,
+      "eval_samples_per_second": 19.447,
+      "eval_steps_per_second": 0.817,
+      "step": 1200
+    },
+    {
+      "epoch": 1.494710640945862,
+      "grad_norm": 2.7152082920074463,
+      "learning_rate": 2.8205719557195576e-06,
+      "loss": 1.4329,
+      "step": 1201
+    },
+    {
+      "epoch": 1.4959551960174238,
+      "grad_norm": 2.866170883178711,
+      "learning_rate": 2.8182656826568266e-06,
+      "loss": 1.4455,
+      "step": 1202
+    },
+    {
+      "epoch": 1.4971997510889856,
+      "grad_norm": 2.9010684490203857,
+      "learning_rate": 2.815959409594096e-06,
+      "loss": 1.423,
+      "step": 1203
+    },
+    {
+      "epoch": 1.4984443061605477,
+      "grad_norm": 2.8588502407073975,
+      "learning_rate": 2.813653136531366e-06,
+      "loss": 1.4041,
+      "step": 1204
+    },
+    {
+      "epoch": 1.4996888612321095,
+      "grad_norm": 2.932544231414795,
+      "learning_rate": 2.811346863468635e-06,
+      "loss": 1.4559,
+      "step": 1205
+    },
+    {
+      "epoch": 1.5009334163036714,
+      "grad_norm": 2.7486014366149902,
+      "learning_rate": 2.8090405904059043e-06,
+      "loss": 1.4337,
+      "step": 1206
+    },
+    {
+      "epoch": 1.5021779713752332,
+      "grad_norm": 3.0866589546203613,
+      "learning_rate": 2.8067343173431737e-06,
+      "loss": 1.4186,
+      "step": 1207
+    },
+    {
+      "epoch": 1.5034225264467953,
+      "grad_norm": 2.9710276126861572,
+      "learning_rate": 2.8044280442804427e-06,
+      "loss": 1.419,
+      "step": 1208
+    },
+    {
+      "epoch": 1.5046670815183572,
+      "grad_norm": 2.8379950523376465,
+      "learning_rate": 2.802121771217712e-06,
+      "loss": 1.4922,
+      "step": 1209
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "grad_norm": 2.939629554748535,
+      "learning_rate": 2.799815498154982e-06,
+      "loss": 1.4402,
+      "step": 1210
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "eval_loss": 1.6419442892074585,
+      "eval_runtime": 43.1202,
+      "eval_samples_per_second": 23.191,
+      "eval_steps_per_second": 0.974,
+      "step": 1210
+    },
+    {
+      "epoch": 1.507156191661481,
+      "grad_norm": 3.041389226913452,
+      "learning_rate": 2.7975092250922514e-06,
+      "loss": 1.4873,
+      "step": 1211
+    },
+    {
+      "epoch": 1.508400746733043,
+      "grad_norm": 2.9776995182037354,
+      "learning_rate": 2.7952029520295204e-06,
+      "loss": 1.4422,
+      "step": 1212
+    },
+    {
+      "epoch": 1.5096453018046048,
+      "grad_norm": 2.8798792362213135,
+      "learning_rate": 2.79289667896679e-06,
+      "loss": 1.4535,
+      "step": 1213
+    },
+    {
+      "epoch": 1.5108898568761666,
+      "grad_norm": 2.9016385078430176,
+      "learning_rate": 2.7905904059040597e-06,
+      "loss": 1.4462,
+      "step": 1214
+    },
+    {
+      "epoch": 1.5121344119477287,
+      "grad_norm": 2.8987581729888916,
+      "learning_rate": 2.7882841328413287e-06,
+      "loss": 1.4477,
+      "step": 1215
+    },
+    {
+      "epoch": 1.5133789670192908,
+      "grad_norm": 2.960266351699829,
+      "learning_rate": 2.785977859778598e-06,
+      "loss": 1.4811,
+      "step": 1216
+    },
+    {
+      "epoch": 1.5146235220908526,
+      "grad_norm": 2.8965611457824707,
+      "learning_rate": 2.7836715867158675e-06,
+      "loss": 1.4366,
+      "step": 1217
+    },
+    {
+      "epoch": 1.5158680771624145,
+      "grad_norm": 3.0667364597320557,
+      "learning_rate": 2.7813653136531365e-06,
+      "loss": 1.3828,
+      "step": 1218
+    },
+    {
+      "epoch": 1.5171126322339763,
+      "grad_norm": 2.952362537384033,
+      "learning_rate": 2.779059040590406e-06,
+      "loss": 1.3903,
+      "step": 1219
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "grad_norm": 2.9899795055389404,
+      "learning_rate": 2.776752767527676e-06,
+      "loss": 1.4312,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "eval_loss": 1.640383005142212,
+      "eval_runtime": 45.4128,
+      "eval_samples_per_second": 22.02,
+      "eval_steps_per_second": 0.925,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5196017423771002,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4344,
+      "step": 1221
+    },
+    {
+      "epoch": 1.520846297448662,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4386,
+      "step": 1222
+    },
+    {
+      "epoch": 1.5220908525202241,
+      "grad_norm": 2.883223295211792,
+      "learning_rate": 2.7721402214022142e-06,
+      "loss": 1.4401,
+      "step": 1223
+    },
+    {
+      "epoch": 1.523335407591786,
+      "grad_norm": 2.8346424102783203,
+      "learning_rate": 2.7698339483394837e-06,
+      "loss": 1.3926,
+      "step": 1224
+    },
+    {
+      "epoch": 1.5245799626633478,
+      "grad_norm": 2.8898870944976807,
+      "learning_rate": 2.767527675276753e-06,
+      "loss": 1.4128,
+      "step": 1225
+    },
+    {
+      "epoch": 1.5258245177349097,
+      "grad_norm": 2.828594923019409,
+      "learning_rate": 2.765221402214022e-06,
+      "loss": 1.4103,
+      "step": 1226
+    },
+    {
+      "epoch": 1.5270690728064715,
+      "grad_norm": 2.882558822631836,
+      "learning_rate": 2.762915129151292e-06,
+      "loss": 1.4186,
+      "step": 1227
+    },
+    {
+      "epoch": 1.5283136278780336,
+      "grad_norm": 2.955760955810547,
+      "learning_rate": 2.7606088560885613e-06,
+      "loss": 1.3705,
+      "step": 1228
+    },
+    {
+      "epoch": 1.5295581829495957,
+      "grad_norm": 2.862934112548828,
+      "learning_rate": 2.7583025830258303e-06,
+      "loss": 1.5012,
+      "step": 1229
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "grad_norm": 2.926501989364624,
+      "learning_rate": 2.7559963099630998e-06,
+      "loss": 1.3875,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "eval_loss": 1.6376254558563232,
+      "eval_runtime": 47.9535,
+      "eval_samples_per_second": 20.854,
+      "eval_steps_per_second": 0.876,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5320472930927194,
+      "grad_norm": 2.7648823261260986,
+      "learning_rate": 2.753690036900369e-06,
+      "loss": 1.3781,
+      "step": 1231
+    },
+    {
+      "epoch": 1.5332918481642812,
+      "grad_norm": 2.927103281021118,
+      "learning_rate": 2.751383763837638e-06,
+      "loss": 1.4091,
+      "step": 1232
+    },
+    {
+      "epoch": 1.534536403235843,
+      "grad_norm": 2.964823007583618,
+      "learning_rate": 2.749077490774908e-06,
+      "loss": 1.5075,
+      "step": 1233
+    },
+    {
+      "epoch": 1.5357809583074051,
+      "grad_norm": 2.7944016456604004,
+      "learning_rate": 2.7467712177121775e-06,
+      "loss": 1.3888,
+      "step": 1234
+    },
+    {
+      "epoch": 1.537025513378967,
+      "grad_norm": 2.891592025756836,
+      "learning_rate": 2.7444649446494465e-06,
+      "loss": 1.4825,
+      "step": 1235
+    },
+    {
+      "epoch": 1.538270068450529,
+      "grad_norm": 2.8120031356811523,
+      "learning_rate": 2.742158671586716e-06,
+      "loss": 1.4246,
+      "step": 1236
+    },
+    {
+      "epoch": 1.539514623522091,
+      "grad_norm": 2.9384255409240723,
+      "learning_rate": 2.7398523985239857e-06,
+      "loss": 1.4657,
+      "step": 1237
+    },
+    {
+      "epoch": 1.5407591785936527,
+      "grad_norm": 2.73111891746521,
+      "learning_rate": 2.737546125461255e-06,
+      "loss": 1.4689,
+      "step": 1238
+    },
+    {
+      "epoch": 1.5420037336652146,
+      "grad_norm": 2.858963966369629,
+      "learning_rate": 2.735239852398524e-06,
+      "loss": 1.4626,
+      "step": 1239
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "grad_norm": 2.9148507118225098,
+      "learning_rate": 2.7329335793357936e-06,
+      "loss": 1.4787,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "eval_loss": 1.6436412334442139,
+      "eval_runtime": 47.3139,
+      "eval_samples_per_second": 21.135,
+      "eval_steps_per_second": 0.888,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5444928438083385,
+      "grad_norm": 2.8731327056884766,
+      "learning_rate": 2.730627306273063e-06,
+      "loss": 1.4748,
+      "step": 1241
+    },
+    {
+      "epoch": 1.5457373988799006,
+      "grad_norm": 2.990265369415283,
+      "learning_rate": 2.728321033210332e-06,
+      "loss": 1.4375,
+      "step": 1242
+    },
+    {
+      "epoch": 1.5469819539514624,
+      "grad_norm": 2.874321460723877,
+      "learning_rate": 2.726014760147602e-06,
+      "loss": 1.4154,
+      "step": 1243
+    },
+    {
+      "epoch": 1.5482265090230243,
+      "grad_norm": 2.9316225051879883,
+      "learning_rate": 2.7237084870848713e-06,
+      "loss": 1.4285,
+      "step": 1244
+    },
+    {
+      "epoch": 1.5494710640945861,
+      "grad_norm": 2.783966302871704,
+      "learning_rate": 2.7214022140221403e-06,
+      "loss": 1.409,
+      "step": 1245
+    },
+    {
+      "epoch": 1.550715619166148,
+      "grad_norm": 2.9578707218170166,
+      "learning_rate": 2.7190959409594097e-06,
+      "loss": 1.4325,
+      "step": 1246
+    },
+    {
+      "epoch": 1.55196017423771,
+      "grad_norm": 2.869893789291382,
+      "learning_rate": 2.716789667896679e-06,
+      "loss": 1.3989,
+      "step": 1247
+    },
+    {
+      "epoch": 1.553204729309272,
+      "grad_norm": 2.962944269180298,
+      "learning_rate": 2.714483394833948e-06,
+      "loss": 1.4302,
+      "step": 1248
+    },
+    {
+      "epoch": 1.554449284380834,
+      "grad_norm": 2.974494695663452,
+      "learning_rate": 2.712177121771218e-06,
+      "loss": 1.46,
+      "step": 1249
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "grad_norm": 2.9154603481292725,
+      "learning_rate": 2.7098708487084874e-06,
+      "loss": 1.4388,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "eval_loss": 1.6357773542404175,
+      "eval_runtime": 47.6085,
+      "eval_samples_per_second": 21.005,
+      "eval_steps_per_second": 0.882,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5569383945239577,
+      "grad_norm": 3.0333547592163086,
+      "learning_rate": 2.707564575645757e-06,
+      "loss": 1.4846,
+      "step": 1251
+    },
+    {
+      "epoch": 1.5581829495955195,
+      "grad_norm": 3.06097674369812,
+      "learning_rate": 2.705258302583026e-06,
+      "loss": 1.4355,
+      "step": 1252
+    },
+    {
+      "epoch": 1.5594275046670814,
+      "grad_norm": 2.872680425643921,
+      "learning_rate": 2.7029520295202956e-06,
+      "loss": 1.4219,
+      "step": 1253
+    },
+    {
+      "epoch": 1.5606720597386434,
+      "grad_norm": 2.9066731929779053,
+      "learning_rate": 2.700645756457565e-06,
+      "loss": 1.384,
+      "step": 1254
+    },
+    {
+      "epoch": 1.5619166148102055,
+      "grad_norm": 3.0570194721221924,
+      "learning_rate": 2.698339483394834e-06,
+      "loss": 1.4687,
+      "step": 1255
+    },
+    {
+      "epoch": 1.5631611698817673,
+      "grad_norm": 3.1955857276916504,
+      "learning_rate": 2.6960332103321035e-06,
+      "loss": 1.4736,
+      "step": 1256
+    },
+    {
+      "epoch": 1.5644057249533292,
+      "grad_norm": 2.9686198234558105,
+      "learning_rate": 2.693726937269373e-06,
+      "loss": 1.4176,
+      "step": 1257
+    },
+    {
+      "epoch": 1.565650280024891,
+      "grad_norm": 3.1161272525787354,
+      "learning_rate": 2.691420664206642e-06,
+      "loss": 1.443,
+      "step": 1258
+    },
+    {
+      "epoch": 1.5668948350964529,
+      "grad_norm": 2.9031193256378174,
+      "learning_rate": 2.6891143911439118e-06,
+      "loss": 1.3871,
+      "step": 1259
+    },
+    {
+      "epoch": 1.568139390168015,
+      "grad_norm": 2.9120476245880127,
+      "learning_rate": 2.686808118081181e-06,
+      "loss": 1.4406,
+      "step": 1260
+    },
+    {
+      "epoch": 1.568139390168015,
+      "eval_loss": 1.6291446685791016,
+      "eval_runtime": 49.7006,
+      "eval_samples_per_second": 20.12,
+      "eval_steps_per_second": 0.845,
+      "step": 1260
+    },
+    {
+      "epoch": 1.5693839452395768,
+      "grad_norm": 3.3181824684143066,
+      "learning_rate": 2.6845018450184506e-06,
+      "loss": 1.5278,
+      "step": 1261
+    },
+    {
+      "epoch": 1.5706285003111389,
+      "grad_norm": 2.9628148078918457,
+      "learning_rate": 2.6821955719557196e-06,
+      "loss": 1.4206,
+      "step": 1262
+    },
+    {
+      "epoch": 1.5718730553827007,
+      "grad_norm": 2.9828474521636963,
+      "learning_rate": 2.679889298892989e-06,
+      "loss": 1.5478,
+      "step": 1263
+    },
+    {
+      "epoch": 1.5731176104542626,
+      "grad_norm": 3.018711805343628,
+      "learning_rate": 2.677583025830259e-06,
+      "loss": 1.4442,
+      "step": 1264
+    },
+    {
+      "epoch": 1.5743621655258244,
+      "grad_norm": 2.8596792221069336,
+      "learning_rate": 2.675276752767528e-06,
+      "loss": 1.4325,
+      "step": 1265
+    },
+    {
+      "epoch": 1.5756067205973865,
+      "grad_norm": 2.9592769145965576,
+      "learning_rate": 2.6729704797047973e-06,
+      "loss": 1.4059,
+      "step": 1266
+    },
+    {
+      "epoch": 1.5768512756689483,
+      "grad_norm": 2.773589611053467,
+      "learning_rate": 2.6706642066420667e-06,
+      "loss": 1.3608,
+      "step": 1267
+    },
+    {
+      "epoch": 1.5780958307405104,
+      "grad_norm": 3.313255548477173,
+      "learning_rate": 2.6683579335793357e-06,
+      "loss": 1.4552,
+      "step": 1268
+    },
+    {
+      "epoch": 1.5793403858120723,
+      "grad_norm": 2.977649211883545,
+      "learning_rate": 2.6660516605166056e-06,
+      "loss": 1.4221,
+      "step": 1269
+    },
+    {
+      "epoch": 1.580584940883634,
+      "grad_norm": 3.0672051906585693,
+      "learning_rate": 2.663745387453875e-06,
+      "loss": 1.4568,
+      "step": 1270
+    },
+    {
+      "epoch": 1.580584940883634,
+      "eval_loss": 1.6317907571792603,
+      "eval_runtime": 47.2354,
+      "eval_samples_per_second": 21.171,
+      "eval_steps_per_second": 0.889,
+      "step": 1270
+    },
+    {
+      "epoch": 1.581829495955196,
+      "grad_norm": 3.029320240020752,
+      "learning_rate": 2.661439114391144e-06,
+      "loss": 1.4432,
+      "step": 1271
+    },
+    {
+      "epoch": 1.5830740510267578,
+      "grad_norm": 3.0183846950531006,
+      "learning_rate": 2.6591328413284134e-06,
+      "loss": 1.4332,
+      "step": 1272
+    },
+    {
+      "epoch": 1.5843186060983199,
+      "grad_norm": 2.856551170349121,
+      "learning_rate": 2.656826568265683e-06,
+      "loss": 1.4291,
+      "step": 1273
+    },
+    {
+      "epoch": 1.5855631611698817,
+      "grad_norm": 3.124971389770508,
+      "learning_rate": 2.6545202952029527e-06,
+      "loss": 1.4363,
+      "step": 1274
+    },
+    {
+      "epoch": 1.5868077162414438,
+      "grad_norm": 2.9470791816711426,
+      "learning_rate": 2.6522140221402217e-06,
+      "loss": 1.4502,
+      "step": 1275
+    },
+    {
+      "epoch": 1.5880522713130056,
+      "grad_norm": 2.9116358757019043,
+      "learning_rate": 2.649907749077491e-06,
+      "loss": 1.4314,
+      "step": 1276
+    },
+    {
+      "epoch": 1.5892968263845675,
+      "grad_norm": 3.2575323581695557,
+      "learning_rate": 2.6476014760147605e-06,
+      "loss": 1.4749,
+      "step": 1277
+    },
+    {
+      "epoch": 1.5905413814561293,
+      "grad_norm": 2.935518503189087,
+      "learning_rate": 2.6452952029520295e-06,
+      "loss": 1.4265,
+      "step": 1278
+    },
+    {
+      "epoch": 1.5917859365276914,
+      "grad_norm": 2.9750072956085205,
+      "learning_rate": 2.642988929889299e-06,
+      "loss": 1.4501,
+      "step": 1279
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "grad_norm": 2.8086845874786377,
+      "learning_rate": 2.640682656826569e-06,
+      "loss": 1.4557,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "eval_loss": 1.6302106380462646,
+      "eval_runtime": 52.3962,
+      "eval_samples_per_second": 19.085,
+      "eval_steps_per_second": 0.802,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5942750466708153,
+      "grad_norm": 2.741849422454834,
+      "learning_rate": 2.638376383763838e-06,
+      "loss": 1.3918,
+      "step": 1281
+    },
+    {
+      "epoch": 1.5955196017423772,
+      "grad_norm": 2.9414806365966797,
+      "learning_rate": 2.6360701107011072e-06,
+      "loss": 1.3882,
+      "step": 1282
+    },
+    {
+      "epoch": 1.596764156813939,
+      "grad_norm": 2.901514768600464,
+      "learning_rate": 2.6337638376383766e-06,
+      "loss": 1.4331,
+      "step": 1283
+    },
+    {
+      "epoch": 1.5980087118855009,
+      "grad_norm": 2.8159122467041016,
+      "learning_rate": 2.6314575645756456e-06,
+      "loss": 1.4064,
+      "step": 1284
+    },
+    {
+      "epoch": 1.5992532669570627,
+      "grad_norm": 2.900561571121216,
+      "learning_rate": 2.629151291512915e-06,
+      "loss": 1.3731,
+      "step": 1285
+    },
+    {
+      "epoch": 1.6004978220286248,
+      "grad_norm": 3.019540548324585,
+      "learning_rate": 2.626845018450185e-06,
+      "loss": 1.4506,
+      "step": 1286
+    },
+    {
+      "epoch": 1.6017423771001866,
+      "grad_norm": 2.9263620376586914,
+      "learning_rate": 2.6245387453874543e-06,
+      "loss": 1.4754,
+      "step": 1287
+    },
+    {
+      "epoch": 1.6029869321717487,
+      "grad_norm": 2.8681881427764893,
+      "learning_rate": 2.6222324723247233e-06,
+      "loss": 1.4124,
+      "step": 1288
+    },
+    {
+      "epoch": 1.6042314872433105,
+      "grad_norm": 2.922945261001587,
+      "learning_rate": 2.6199261992619928e-06,
+      "loss": 1.5101,
+      "step": 1289
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "grad_norm": 2.9548606872558594,
+      "learning_rate": 2.6176199261992626e-06,
+      "loss": 1.41,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "eval_loss": 1.6357502937316895,
+      "eval_runtime": 46.9547,
+      "eval_samples_per_second": 21.297,
+      "eval_steps_per_second": 0.894,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6067205973864342,
+      "grad_norm": 2.837184190750122,
+      "learning_rate": 2.6153136531365316e-06,
+      "loss": 1.4354,
+      "step": 1291
+    },
+    {
+      "epoch": 1.6079651524579963,
+      "grad_norm": 2.818990468978882,
+      "learning_rate": 2.613007380073801e-06,
+      "loss": 1.4474,
+      "step": 1292
+    },
+    {
+      "epoch": 1.6092097075295582,
+      "grad_norm": 2.7880146503448486,
+      "learning_rate": 2.6107011070110704e-06,
+      "loss": 1.4156,
+      "step": 1293
+    },
+    {
+      "epoch": 1.6104542626011202,
+      "grad_norm": 2.790971040725708,
+      "learning_rate": 2.6083948339483394e-06,
+      "loss": 1.4671,
+      "step": 1294
+    },
+    {
+      "epoch": 1.611698817672682,
+      "grad_norm": 2.943129777908325,
+      "learning_rate": 2.606088560885609e-06,
+      "loss": 1.4801,
+      "step": 1295
+    },
+    {
+      "epoch": 1.612943372744244,
+      "grad_norm": 2.8589110374450684,
+      "learning_rate": 2.6037822878228787e-06,
+      "loss": 1.4894,
+      "step": 1296
+    },
+    {
+      "epoch": 1.6141879278158058,
+      "grad_norm": 2.8929262161254883,
+      "learning_rate": 2.6014760147601477e-06,
+      "loss": 1.4022,
+      "step": 1297
+    },
+    {
+      "epoch": 1.6154324828873676,
+      "grad_norm": 2.970099925994873,
+      "learning_rate": 2.599169741697417e-06,
+      "loss": 1.5136,
+      "step": 1298
+    },
+    {
+      "epoch": 1.6166770379589297,
+      "grad_norm": 2.928466796875,
+      "learning_rate": 2.5968634686346866e-06,
+      "loss": 1.4743,
+      "step": 1299
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "grad_norm": 2.941580057144165,
+      "learning_rate": 2.594557195571956e-06,
+      "loss": 1.4397,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "eval_loss": 1.6348339319229126,
+      "eval_runtime": 50.433,
+      "eval_samples_per_second": 19.828,
+      "eval_steps_per_second": 0.833,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6191661481020536,
+      "grad_norm": 2.903212308883667,
+      "learning_rate": 2.592250922509225e-06,
+      "loss": 1.4221,
+      "step": 1301
+    },
+    {
+      "epoch": 1.6204107031736155,
+      "grad_norm": 2.948651075363159,
+      "learning_rate": 2.589944649446495e-06,
+      "loss": 1.4363,
+      "step": 1302
+    },
+    {
+      "epoch": 1.6216552582451773,
+      "grad_norm": 2.8940505981445312,
+      "learning_rate": 2.5876383763837643e-06,
+      "loss": 1.398,
+      "step": 1303
+    },
+    {
+      "epoch": 1.6228998133167392,
+      "grad_norm": 2.950056552886963,
+      "learning_rate": 2.5853321033210333e-06,
+      "loss": 1.4425,
+      "step": 1304
+    },
+    {
+      "epoch": 1.6241443683883012,
+      "grad_norm": 2.9674088954925537,
+      "learning_rate": 2.5830258302583027e-06,
+      "loss": 1.4473,
+      "step": 1305
+    },
+    {
+      "epoch": 1.625388923459863,
+      "grad_norm": 2.7541685104370117,
+      "learning_rate": 2.5807195571955725e-06,
+      "loss": 1.4483,
+      "step": 1306
+    },
+    {
+      "epoch": 1.6266334785314251,
+      "grad_norm": 2.8897035121917725,
+      "learning_rate": 2.5784132841328415e-06,
+      "loss": 1.4572,
+      "step": 1307
+    },
+    {
+      "epoch": 1.627878033602987,
+      "grad_norm": 3.021559238433838,
+      "learning_rate": 2.576107011070111e-06,
+      "loss": 1.5164,
+      "step": 1308
+    },
+    {
+      "epoch": 1.6291225886745488,
+      "grad_norm": 2.884009838104248,
+      "learning_rate": 2.5738007380073804e-06,
+      "loss": 1.4196,
+      "step": 1309
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "grad_norm": 2.749497890472412,
+      "learning_rate": 2.5714944649446494e-06,
+      "loss": 1.4324,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "eval_loss": 1.6314265727996826,
+      "eval_runtime": 42.4647,
+      "eval_samples_per_second": 23.549,
+      "eval_steps_per_second": 0.989,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6316116988176725,
+      "grad_norm": 2.8908045291900635,
+      "learning_rate": 2.5691881918819188e-06,
+      "loss": 1.4388,
+      "step": 1311
+    },
+    {
+      "epoch": 1.6328562538892346,
+      "grad_norm": 2.832808256149292,
+      "learning_rate": 2.5668819188191886e-06,
+      "loss": 1.4368,
+      "step": 1312
+    },
+    {
+      "epoch": 1.6341008089607967,
+      "grad_norm": 2.9047906398773193,
+      "learning_rate": 2.564575645756458e-06,
+      "loss": 1.4186,
+      "step": 1313
+    },
+    {
+      "epoch": 1.6353453640323585,
+      "grad_norm": 2.9271419048309326,
+      "learning_rate": 2.562269372693727e-06,
+      "loss": 1.3776,
+      "step": 1314
+    },
+    {
+      "epoch": 1.6365899191039204,
+      "grad_norm": 2.760314702987671,
+      "learning_rate": 2.5599630996309965e-06,
+      "loss": 1.386,
+      "step": 1315
+    },
+    {
+      "epoch": 1.6378344741754822,
+      "grad_norm": 2.9694864749908447,
+      "learning_rate": 2.557656826568266e-06,
+      "loss": 1.4542,
+      "step": 1316
+    },
+    {
+      "epoch": 1.639079029247044,
+      "grad_norm": 2.9950435161590576,
+      "learning_rate": 2.555350553505535e-06,
+      "loss": 1.4242,
+      "step": 1317
+    },
+    {
+      "epoch": 1.6403235843186061,
+      "grad_norm": 2.9291114807128906,
+      "learning_rate": 2.5530442804428047e-06,
+      "loss": 1.4289,
+      "step": 1318
+    },
+    {
+      "epoch": 1.641568139390168,
+      "grad_norm": 2.981961488723755,
+      "learning_rate": 2.550738007380074e-06,
+      "loss": 1.4283,
+      "step": 1319
+    },
+    {
+      "epoch": 1.64281269446173,
+      "grad_norm": 2.734123706817627,
+      "learning_rate": 2.548431734317343e-06,
+      "loss": 1.4144,
+      "step": 1320
+    },
+    {
+      "epoch": 1.64281269446173,
+      "eval_loss": 1.6249239444732666,
+      "eval_runtime": 48.8116,
+      "eval_samples_per_second": 20.487,
+      "eval_steps_per_second": 0.86,
+      "step": 1320
+    },
+    {
+      "epoch": 1.644057249533292,
+      "grad_norm": 2.9752919673919678,
+      "learning_rate": 2.5461254612546126e-06,
+      "loss": 1.3641,
+      "step": 1321
+    },
+    {
+      "epoch": 1.6453018046048538,
+      "grad_norm": 2.972463607788086,
+      "learning_rate": 2.543819188191882e-06,
+      "loss": 1.3797,
+      "step": 1322
+    },
+    {
+      "epoch": 1.6465463596764156,
+      "grad_norm": 2.905499219894409,
+      "learning_rate": 2.541512915129152e-06,
+      "loss": 1.4221,
+      "step": 1323
+    },
+    {
+      "epoch": 1.6477909147479775,
+      "grad_norm": 2.7921793460845947,
+      "learning_rate": 2.539206642066421e-06,
+      "loss": 1.4118,
+      "step": 1324
+    },
+    {
+      "epoch": 1.6490354698195395,
+      "grad_norm": 3.064558267593384,
+      "learning_rate": 2.5369003690036903e-06,
+      "loss": 1.4611,
+      "step": 1325
+    },
+    {
+      "epoch": 1.6502800248911016,
+      "grad_norm": 2.9690096378326416,
+      "learning_rate": 2.5345940959409597e-06,
+      "loss": 1.461,
+      "step": 1326
+    },
+    {
+      "epoch": 1.6515245799626634,
+      "grad_norm": 3.0379040241241455,
+      "learning_rate": 2.5322878228782287e-06,
+      "loss": 1.451,
+      "step": 1327
+    },
+    {
+      "epoch": 1.6527691350342253,
+      "grad_norm": 2.9426493644714355,
+      "learning_rate": 2.5299815498154986e-06,
+      "loss": 1.4671,
+      "step": 1328
+    },
+    {
+      "epoch": 1.6540136901057871,
+      "grad_norm": 2.915076732635498,
+      "learning_rate": 2.527675276752768e-06,
+      "loss": 1.4159,
+      "step": 1329
+    },
+    {
+      "epoch": 1.655258245177349,
+      "grad_norm": 2.8588879108428955,
+      "learning_rate": 2.525369003690037e-06,
+      "loss": 1.4978,
+      "step": 1330
+    },
+    {
+      "epoch": 1.655258245177349,
+      "eval_loss": 1.624009132385254,
+      "eval_runtime": 47.0461,
+      "eval_samples_per_second": 21.256,
+      "eval_steps_per_second": 0.893,
+      "step": 1330
+    },
+    {
+      "epoch": 1.656502800248911,
+      "grad_norm": 2.9942209720611572,
+      "learning_rate": 2.5230627306273064e-06,
+      "loss": 1.3872,
+      "step": 1331
+    },
+    {
+      "epoch": 1.657747355320473,
+      "grad_norm": 2.8424410820007324,
+      "learning_rate": 2.520756457564576e-06,
+      "loss": 1.4181,
+      "step": 1332
+    },
+    {
+      "epoch": 1.658991910392035,
+      "grad_norm": 2.9234843254089355,
+      "learning_rate": 2.518450184501845e-06,
+      "loss": 1.4516,
+      "step": 1333
+    },
+    {
+      "epoch": 1.6602364654635968,
+      "grad_norm": 2.8367815017700195,
+      "learning_rate": 2.5161439114391147e-06,
+      "loss": 1.4214,
+      "step": 1334
+    },
+    {
+      "epoch": 1.6614810205351587,
+      "grad_norm": 2.819149971008301,
+      "learning_rate": 2.513837638376384e-06,
+      "loss": 1.3883,
+      "step": 1335
+    },
+    {
+      "epoch": 1.6627255756067205,
+      "grad_norm": 3.024693250656128,
+      "learning_rate": 2.5115313653136535e-06,
+      "loss": 1.3902,
+      "step": 1336
+    },
+    {
+      "epoch": 1.6639701306782824,
+      "grad_norm": 2.903599262237549,
+      "learning_rate": 2.5092250922509225e-06,
+      "loss": 1.4238,
+      "step": 1337
+    },
+    {
+      "epoch": 1.6652146857498444,
+      "grad_norm": 2.8296589851379395,
+      "learning_rate": 2.506918819188192e-06,
+      "loss": 1.3993,
+      "step": 1338
+    },
+    {
+      "epoch": 1.6664592408214065,
+      "grad_norm": 2.873272657394409,
+      "learning_rate": 2.5046125461254618e-06,
+      "loss": 1.4186,
+      "step": 1339
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "grad_norm": 2.9268789291381836,
+      "learning_rate": 2.5023062730627308e-06,
+      "loss": 1.4242,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "eval_loss": 1.6260672807693481,
+      "eval_runtime": 47.5895,
+      "eval_samples_per_second": 21.013,
+      "eval_steps_per_second": 0.883,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6689483509645302,
+      "grad_norm": 3.057659387588501,
+      "learning_rate": 2.5e-06,
+      "loss": 1.4077,
+      "step": 1341
+    },
+    {
+      "epoch": 1.670192906036092,
+      "grad_norm": 2.875095844268799,
+      "learning_rate": 2.4976937269372696e-06,
+      "loss": 1.394,
+      "step": 1342
+    },
+    {
+      "epoch": 1.671437461107654,
+      "grad_norm": 2.7579872608184814,
+      "learning_rate": 2.495387453874539e-06,
+      "loss": 1.4029,
+      "step": 1343
+    },
+    {
+      "epoch": 1.672682016179216,
+      "grad_norm": 2.8981001377105713,
+      "learning_rate": 2.4930811808118085e-06,
+      "loss": 1.4494,
+      "step": 1344
+    },
+    {
+      "epoch": 1.6739265712507778,
+      "grad_norm": 2.9696900844573975,
+      "learning_rate": 2.490774907749078e-06,
+      "loss": 1.4692,
+      "step": 1345
+    },
+    {
+      "epoch": 1.6751711263223399,
+      "grad_norm": 2.8094892501831055,
+      "learning_rate": 2.488468634686347e-06,
+      "loss": 1.441,
+      "step": 1346
+    },
+    {
+      "epoch": 1.6764156813939017,
+      "grad_norm": 2.8619563579559326,
+      "learning_rate": 2.4861623616236163e-06,
+      "loss": 1.4885,
+      "step": 1347
+    },
+    {
+      "epoch": 1.6776602364654636,
+      "grad_norm": 2.854151487350464,
+      "learning_rate": 2.4838560885608857e-06,
+      "loss": 1.4338,
+      "step": 1348
+    },
+    {
+      "epoch": 1.6789047915370254,
+      "grad_norm": 2.9707868099212646,
+      "learning_rate": 2.481549815498155e-06,
+      "loss": 1.4529,
+      "step": 1349
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "grad_norm": 3.0289969444274902,
+      "learning_rate": 2.4792435424354246e-06,
+      "loss": 1.4855,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "eval_loss": 1.6256393194198608,
+      "eval_runtime": 45.9524,
+      "eval_samples_per_second": 21.762,
+      "eval_steps_per_second": 0.914,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6813939016801493,
+      "grad_norm": 2.8407845497131348,
+      "learning_rate": 2.476937269372694e-06,
+      "loss": 1.3876,
+      "step": 1351
+    },
+    {
+      "epoch": 1.6826384567517114,
+      "grad_norm": 2.9876599311828613,
+      "learning_rate": 2.4746309963099634e-06,
+      "loss": 1.3931,
+      "step": 1352
+    },
+    {
+      "epoch": 1.6838830118232733,
+      "grad_norm": 3.008683919906616,
+      "learning_rate": 2.472324723247233e-06,
+      "loss": 1.4482,
+      "step": 1353
+    },
+    {
+      "epoch": 1.6851275668948351,
+      "grad_norm": 2.8110079765319824,
+      "learning_rate": 2.470018450184502e-06,
+      "loss": 1.3851,
+      "step": 1354
+    },
+    {
+      "epoch": 1.686372121966397,
+      "grad_norm": 2.9402263164520264,
+      "learning_rate": 2.4677121771217713e-06,
+      "loss": 1.4009,
+      "step": 1355
+    },
+    {
+      "epoch": 1.6876166770379588,
+      "grad_norm": 2.9846489429473877,
+      "learning_rate": 2.4654059040590407e-06,
+      "loss": 1.3975,
+      "step": 1356
+    },
+    {
+      "epoch": 1.6888612321095209,
+      "grad_norm": 2.835801362991333,
+      "learning_rate": 2.46309963099631e-06,
+      "loss": 1.3421,
+      "step": 1357
+    },
+    {
+      "epoch": 1.6901057871810827,
+      "grad_norm": 2.9438889026641846,
+      "learning_rate": 2.4607933579335795e-06,
+      "loss": 1.4422,
+      "step": 1358
+    },
+    {
+      "epoch": 1.6913503422526448,
+      "grad_norm": 3.168829917907715,
+      "learning_rate": 2.458487084870849e-06,
+      "loss": 1.4596,
+      "step": 1359
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "grad_norm": 2.8412039279937744,
+      "learning_rate": 2.4561808118081184e-06,
+      "loss": 1.4263,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "eval_loss": 1.6265885829925537,
+      "eval_runtime": 50.3402,
+      "eval_samples_per_second": 19.865,
+      "eval_steps_per_second": 0.834,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6938394523957685,
+      "grad_norm": 3.0540060997009277,
+      "learning_rate": 2.453874538745388e-06,
+      "loss": 1.4123,
+      "step": 1361
+    },
+    {
+      "epoch": 1.6950840074673303,
+      "grad_norm": 3.0544989109039307,
+      "learning_rate": 2.451568265682657e-06,
+      "loss": 1.412,
+      "step": 1362
+    },
+    {
+      "epoch": 1.6963285625388922,
+      "grad_norm": 3.114126205444336,
+      "learning_rate": 2.4492619926199267e-06,
+      "loss": 1.4952,
+      "step": 1363
+    },
+    {
+      "epoch": 1.6975731176104543,
+      "grad_norm": 2.9366016387939453,
+      "learning_rate": 2.4469557195571957e-06,
+      "loss": 1.5155,
+      "step": 1364
+    },
+    {
+      "epoch": 1.6988176726820163,
+      "grad_norm": 2.910677671432495,
+      "learning_rate": 2.444649446494465e-06,
+      "loss": 1.4393,
+      "step": 1365
+    },
+    {
+      "epoch": 1.7000622277535782,
+      "grad_norm": 2.8373727798461914,
+      "learning_rate": 2.4423431734317345e-06,
+      "loss": 1.4223,
+      "step": 1366
+    },
+    {
+      "epoch": 1.70130678282514,
+      "grad_norm": 2.865161895751953,
+      "learning_rate": 2.440036900369004e-06,
+      "loss": 1.4605,
+      "step": 1367
+    },
+    {
+      "epoch": 1.7025513378967019,
+      "grad_norm": 2.7929177284240723,
+      "learning_rate": 2.437730627306273e-06,
+      "loss": 1.4695,
+      "step": 1368
+    },
+    {
+      "epoch": 1.7037958929682637,
+      "grad_norm": 2.867953300476074,
+      "learning_rate": 2.4354243542435428e-06,
+      "loss": 1.3524,
+      "step": 1369
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "grad_norm": 2.8980207443237305,
+      "learning_rate": 2.4331180811808118e-06,
+      "loss": 1.4568,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "eval_loss": 1.6297556161880493,
+      "eval_runtime": 51.447,
+      "eval_samples_per_second": 19.437,
+      "eval_steps_per_second": 0.816,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7062850031113876,
+      "grad_norm": 2.954972505569458,
+      "learning_rate": 2.4308118081180816e-06,
+      "loss": 1.4,
+      "step": 1371
+    },
+    {
+      "epoch": 1.7075295581829497,
+      "grad_norm": 2.973191738128662,
+      "learning_rate": 2.4285055350553506e-06,
+      "loss": 1.4062,
+      "step": 1372
+    },
+    {
+      "epoch": 1.7087741132545116,
+      "grad_norm": 2.951444387435913,
+      "learning_rate": 2.42619926199262e-06,
+      "loss": 1.4523,
+      "step": 1373
+    },
+    {
+      "epoch": 1.7100186683260734,
+      "grad_norm": 2.850445508956909,
+      "learning_rate": 2.4238929889298895e-06,
+      "loss": 1.4282,
+      "step": 1374
+    },
+    {
+      "epoch": 1.7112632233976353,
+      "grad_norm": 2.7643561363220215,
+      "learning_rate": 2.421586715867159e-06,
+      "loss": 1.3807,
+      "step": 1375
+    },
+    {
+      "epoch": 1.712507778469197,
+      "grad_norm": 2.8872134685516357,
+      "learning_rate": 2.4192804428044283e-06,
+      "loss": 1.4801,
+      "step": 1376
+    },
+    {
+      "epoch": 1.7137523335407592,
+      "grad_norm": 2.8748483657836914,
+      "learning_rate": 2.4169741697416977e-06,
+      "loss": 1.381,
+      "step": 1377
+    },
+    {
+      "epoch": 1.7149968886123212,
+      "grad_norm": 2.7833831310272217,
+      "learning_rate": 2.4146678966789667e-06,
+      "loss": 1.389,
+      "step": 1378
+    },
+    {
+      "epoch": 1.716241443683883,
+      "grad_norm": 2.994715690612793,
+      "learning_rate": 2.4123616236162366e-06,
+      "loss": 1.3917,
+      "step": 1379
+    },
+    {
+      "epoch": 1.717485998755445,
+      "grad_norm": 2.887026786804199,
+      "learning_rate": 2.4100553505535056e-06,
+      "loss": 1.3725,
+      "step": 1380
+    },
+    {
+      "epoch": 1.717485998755445,
+      "eval_loss": 1.6283434629440308,
+      "eval_runtime": 50.596,
+      "eval_samples_per_second": 19.764,
+      "eval_steps_per_second": 0.83,
+      "step": 1380
+    },
+    {
+      "epoch": 1.7187305538270068,
+      "grad_norm": 3.2046546936035156,
+      "learning_rate": 2.407749077490775e-06,
+      "loss": 1.4422,
+      "step": 1381
+    },
+    {
+      "epoch": 1.7199751088985686,
+      "grad_norm": 3.0105698108673096,
+      "learning_rate": 2.4054428044280444e-06,
+      "loss": 1.4494,
+      "step": 1382
+    },
+    {
+      "epoch": 1.7212196639701307,
+      "grad_norm": 2.9917800426483154,
+      "learning_rate": 2.403136531365314e-06,
+      "loss": 1.4407,
+      "step": 1383
+    },
+    {
+      "epoch": 1.7224642190416926,
+      "grad_norm": 2.844325065612793,
+      "learning_rate": 2.4008302583025833e-06,
+      "loss": 1.4067,
+      "step": 1384
+    },
+    {
+      "epoch": 1.7237087741132546,
+      "grad_norm": 2.919576406478882,
+      "learning_rate": 2.3985239852398527e-06,
+      "loss": 1.4352,
+      "step": 1385
+    },
+    {
+      "epoch": 1.7249533291848165,
+      "grad_norm": 2.8357744216918945,
+      "learning_rate": 2.3962177121771217e-06,
+      "loss": 1.3978,
+      "step": 1386
+    },
+    {
+      "epoch": 1.7261978842563783,
+      "grad_norm": 2.82425856590271,
+      "learning_rate": 2.3939114391143915e-06,
+      "loss": 1.4439,
+      "step": 1387
+    },
+    {
+      "epoch": 1.7274424393279402,
+      "grad_norm": 2.9155638217926025,
+      "learning_rate": 2.3916051660516605e-06,
+      "loss": 1.4326,
+      "step": 1388
+    },
+    {
+      "epoch": 1.728686994399502,
+      "grad_norm": 2.862804651260376,
+      "learning_rate": 2.3892988929889304e-06,
+      "loss": 1.3766,
+      "step": 1389
+    },
+    {
+      "epoch": 1.729931549471064,
+      "grad_norm": 3.012230634689331,
+      "learning_rate": 2.3869926199261994e-06,
+      "loss": 1.4569,
+      "step": 1390
+    },
+    {
+      "epoch": 1.729931549471064,
+      "eval_loss": 1.6268510818481445,
+      "eval_runtime": 52.5438,
+      "eval_samples_per_second": 19.032,
+      "eval_steps_per_second": 0.799,
+      "step": 1390
+    },
+    {
+      "epoch": 1.7311761045426262,
+      "grad_norm": 2.9922425746917725,
+      "learning_rate": 2.384686346863469e-06,
+      "loss": 1.4066,
+      "step": 1391
+    },
+    {
+      "epoch": 1.732420659614188,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 2.3823800738007382e-06,
+      "loss": 1.4514,
+      "step": 1392
+    },
+    {
+      "epoch": 1.7336652146857499,
+      "grad_norm": 2.834998846054077,
+      "learning_rate": 2.3800738007380077e-06,
+      "loss": 1.4714,
+      "step": 1393
+    },
+    {
+      "epoch": 1.7349097697573117,
+      "grad_norm": 2.9646077156066895,
+      "learning_rate": 2.377767527675277e-06,
+      "loss": 1.4368,
+      "step": 1394
+    },
+    {
+      "epoch": 1.7361543248288736,
+      "grad_norm": 2.8233699798583984,
+      "learning_rate": 2.3754612546125465e-06,
+      "loss": 1.4598,
+      "step": 1395
+    },
+    {
+      "epoch": 1.7373988799004356,
+      "grad_norm": 2.895472764968872,
+      "learning_rate": 2.3731549815498155e-06,
+      "loss": 1.4551,
+      "step": 1396
+    },
+    {
+      "epoch": 1.7386434349719975,
+      "grad_norm": 2.7917752265930176,
+      "learning_rate": 2.3708487084870853e-06,
+      "loss": 1.4759,
+      "step": 1397
+    },
+    {
+      "epoch": 1.7398879900435595,
+      "grad_norm": 2.820672035217285,
+      "learning_rate": 2.3685424354243543e-06,
+      "loss": 1.3711,
+      "step": 1398
+    },
+    {
+      "epoch": 1.7411325451151214,
+      "grad_norm": 2.804952621459961,
+      "learning_rate": 2.3662361623616238e-06,
+      "loss": 1.4227,
+      "step": 1399
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "grad_norm": 2.981553316116333,
+      "learning_rate": 2.363929889298893e-06,
+      "loss": 1.4639,
+      "step": 1400
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "eval_loss": 1.6229957342147827,
+      "eval_runtime": 49.1507,
+      "eval_samples_per_second": 20.346,
+      "eval_steps_per_second": 0.855,
+      "step": 1400
+    },
+    {
+      "epoch": 1.743621655258245,
+      "grad_norm": 2.8649392127990723,
+      "learning_rate": 2.3616236162361626e-06,
+      "loss": 1.4283,
+      "step": 1401
+    },
+    {
+      "epoch": 1.744866210329807,
+      "grad_norm": 2.9438977241516113,
+      "learning_rate": 2.359317343173432e-06,
+      "loss": 1.444,
+      "step": 1402
+    },
+    {
+      "epoch": 1.746110765401369,
+      "grad_norm": 2.9270176887512207,
+      "learning_rate": 2.3570110701107015e-06,
+      "loss": 1.4044,
+      "step": 1403
+    },
+    {
+      "epoch": 1.747355320472931,
+      "grad_norm": 2.791527271270752,
+      "learning_rate": 2.3547047970479705e-06,
+      "loss": 1.4195,
+      "step": 1404
+    },
+    {
+      "epoch": 1.748599875544493,
+      "grad_norm": 2.8519608974456787,
+      "learning_rate": 2.3523985239852403e-06,
+      "loss": 1.4031,
+      "step": 1405
+    },
+    {
+      "epoch": 1.7498444306160548,
+      "grad_norm": 2.90872859954834,
+      "learning_rate": 2.3500922509225093e-06,
+      "loss": 1.4616,
+      "step": 1406
+    },
+    {
+      "epoch": 1.7510889856876166,
+      "grad_norm": 2.9190874099731445,
+      "learning_rate": 2.3477859778597787e-06,
+      "loss": 1.3917,
+      "step": 1407
+    },
+    {
+      "epoch": 1.7523335407591785,
+      "grad_norm": 2.868323802947998,
+      "learning_rate": 2.345479704797048e-06,
+      "loss": 1.4053,
+      "step": 1408
+    },
+    {
+      "epoch": 1.7535780958307405,
+      "grad_norm": 2.8638579845428467,
+      "learning_rate": 2.3431734317343176e-06,
+      "loss": 1.4042,
+      "step": 1409
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "grad_norm": 3.1759817600250244,
+      "learning_rate": 2.340867158671587e-06,
+      "loss": 1.4937,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "eval_loss": 1.6252617835998535,
+      "eval_runtime": 44.3041,
+      "eval_samples_per_second": 22.571,
+      "eval_steps_per_second": 0.948,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7560672059738645,
+      "grad_norm": 2.8265459537506104,
+      "learning_rate": 2.3385608856088564e-06,
+      "loss": 1.406,
+      "step": 1411
+    },
+    {
+      "epoch": 1.7573117610454263,
+      "grad_norm": 2.856229305267334,
+      "learning_rate": 2.336254612546126e-06,
+      "loss": 1.4395,
+      "step": 1412
+    },
+    {
+      "epoch": 1.7585563161169881,
+      "grad_norm": 2.9112775325775146,
+      "learning_rate": 2.333948339483395e-06,
+      "loss": 1.427,
+      "step": 1413
+    },
+    {
+      "epoch": 1.75980087118855,
+      "grad_norm": 3.098846435546875,
+      "learning_rate": 2.3316420664206643e-06,
+      "loss": 1.4222,
+      "step": 1414
+    },
+    {
+      "epoch": 1.7610454262601118,
+      "grad_norm": 2.758100986480713,
+      "learning_rate": 2.3293357933579337e-06,
+      "loss": 1.3844,
+      "step": 1415
+    },
+    {
+      "epoch": 1.762289981331674,
+      "grad_norm": 2.8422274589538574,
+      "learning_rate": 2.327029520295203e-06,
+      "loss": 1.4033,
+      "step": 1416
+    },
+    {
+      "epoch": 1.763534536403236,
+      "grad_norm": 2.9257912635803223,
+      "learning_rate": 2.3247232472324725e-06,
+      "loss": 1.3961,
+      "step": 1417
+    },
+    {
+      "epoch": 1.7647790914747978,
+      "grad_norm": 2.9431657791137695,
+      "learning_rate": 2.322416974169742e-06,
+      "loss": 1.4451,
+      "step": 1418
+    },
+    {
+      "epoch": 1.7660236465463597,
+      "grad_norm": 2.8598814010620117,
+      "learning_rate": 2.3201107011070114e-06,
+      "loss": 1.3816,
+      "step": 1419
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "grad_norm": 2.9868006706237793,
+      "learning_rate": 2.317804428044281e-06,
+      "loss": 1.4037,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "eval_loss": 1.6231273412704468,
+      "eval_runtime": 48.8067,
+      "eval_samples_per_second": 20.489,
+      "eval_steps_per_second": 0.861,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7685127566894834,
+      "grad_norm": 2.888206720352173,
+      "learning_rate": 2.31549815498155e-06,
+      "loss": 1.4087,
+      "step": 1421
+    },
+    {
+      "epoch": 1.7697573117610454,
+      "grad_norm": 2.9240078926086426,
+      "learning_rate": 2.3131918819188192e-06,
+      "loss": 1.3909,
+      "step": 1422
+    },
+    {
+      "epoch": 1.7710018668326073,
+      "grad_norm": 3.0401012897491455,
+      "learning_rate": 2.3108856088560886e-06,
+      "loss": 1.423,
+      "step": 1423
+    },
+    {
+      "epoch": 1.7722464219041694,
+      "grad_norm": 2.9484200477600098,
+      "learning_rate": 2.308579335793358e-06,
+      "loss": 1.449,
+      "step": 1424
+    },
+    {
+      "epoch": 1.7734909769757312,
+      "grad_norm": 2.863267660140991,
+      "learning_rate": 2.3062730627306275e-06,
+      "loss": 1.4215,
+      "step": 1425
+    },
+    {
+      "epoch": 1.774735532047293,
+      "grad_norm": 2.9615612030029297,
+      "learning_rate": 2.303966789667897e-06,
+      "loss": 1.4391,
+      "step": 1426
+    },
+    {
+      "epoch": 1.775980087118855,
+      "grad_norm": 2.7958569526672363,
+      "learning_rate": 2.3016605166051663e-06,
+      "loss": 1.3662,
+      "step": 1427
+    },
+    {
+      "epoch": 1.777224642190417,
+      "grad_norm": 2.898787021636963,
+      "learning_rate": 2.2993542435424358e-06,
+      "loss": 1.362,
+      "step": 1428
+    },
+    {
+      "epoch": 1.7784691972619788,
+      "grad_norm": 2.883049726486206,
+      "learning_rate": 2.2970479704797048e-06,
+      "loss": 1.4339,
+      "step": 1429
+    },
+    {
+      "epoch": 1.779713752333541,
+      "grad_norm": 3.0204827785491943,
+      "learning_rate": 2.294741697416974e-06,
+      "loss": 1.5174,
+      "step": 1430
+    },
+    {
+      "epoch": 1.779713752333541,
+      "eval_loss": 1.6246228218078613,
+      "eval_runtime": 47.5611,
+      "eval_samples_per_second": 21.026,
+      "eval_steps_per_second": 0.883,
+      "step": 1430
+    },
+    {
+      "epoch": 1.7809583074051027,
+      "grad_norm": 2.9354310035705566,
+      "learning_rate": 2.2924354243542436e-06,
+      "loss": 1.4115,
+      "step": 1431
+    },
+    {
+      "epoch": 1.7822028624766646,
+      "grad_norm": 2.9498608112335205,
+      "learning_rate": 2.290129151291513e-06,
+      "loss": 1.4058,
+      "step": 1432
+    },
+    {
+      "epoch": 1.7834474175482264,
+      "grad_norm": 2.9706921577453613,
+      "learning_rate": 2.2878228782287825e-06,
+      "loss": 1.426,
+      "step": 1433
+    },
+    {
+      "epoch": 1.7846919726197883,
+      "grad_norm": 2.8945024013519287,
+      "learning_rate": 2.285516605166052e-06,
+      "loss": 1.4011,
+      "step": 1434
+    },
+    {
+      "epoch": 1.7859365276913504,
+      "grad_norm": 2.8987081050872803,
+      "learning_rate": 2.2832103321033213e-06,
+      "loss": 1.4163,
+      "step": 1435
+    },
+    {
+      "epoch": 1.7871810827629122,
+      "grad_norm": 2.9797592163085938,
+      "learning_rate": 2.2809040590405907e-06,
+      "loss": 1.4367,
+      "step": 1436
+    },
+    {
+      "epoch": 1.7884256378344743,
+      "grad_norm": 2.8872063159942627,
+      "learning_rate": 2.2785977859778597e-06,
+      "loss": 1.4226,
+      "step": 1437
+    },
+    {
+      "epoch": 1.7896701929060361,
+      "grad_norm": 2.942655563354492,
+      "learning_rate": 2.2762915129151296e-06,
+      "loss": 1.4132,
+      "step": 1438
+    },
+    {
+      "epoch": 1.790914747977598,
+      "grad_norm": 3.027392625808716,
+      "learning_rate": 2.2739852398523986e-06,
+      "loss": 1.4148,
+      "step": 1439
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "grad_norm": 2.9475181102752686,
+      "learning_rate": 2.271678966789668e-06,
+      "loss": 1.4585,
+      "step": 1440
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "eval_loss": 1.6234897375106812,
+      "eval_runtime": 53.3737,
+      "eval_samples_per_second": 18.736,
+      "eval_steps_per_second": 0.787,
+      "step": 1440
+    },
+    {
+      "epoch": 1.793403858120722,
+      "grad_norm": 2.846304416656494,
+      "learning_rate": 2.2693726937269374e-06,
+      "loss": 1.3667,
+      "step": 1441
+    },
+    {
+      "epoch": 1.7946484131922837,
+      "grad_norm": 2.8925700187683105,
+      "learning_rate": 2.267066420664207e-06,
+      "loss": 1.4318,
+      "step": 1442
+    },
+    {
+      "epoch": 1.7958929682638458,
+      "grad_norm": 2.884025812149048,
+      "learning_rate": 2.2647601476014763e-06,
+      "loss": 1.4395,
+      "step": 1443
+    },
+    {
+      "epoch": 1.7971375233354077,
+      "grad_norm": 2.9763753414154053,
+      "learning_rate": 2.2624538745387457e-06,
+      "loss": 1.3742,
+      "step": 1444
+    },
+    {
+      "epoch": 1.7983820784069695,
+      "grad_norm": 3.0834898948669434,
+      "learning_rate": 2.2601476014760147e-06,
+      "loss": 1.4651,
+      "step": 1445
+    },
+    {
+      "epoch": 1.7996266334785314,
+      "grad_norm": 3.0673978328704834,
+      "learning_rate": 2.2578413284132845e-06,
+      "loss": 1.475,
+      "step": 1446
+    },
+    {
+      "epoch": 1.8008711885500932,
+      "grad_norm": 2.931521415710449,
+      "learning_rate": 2.2555350553505535e-06,
+      "loss": 1.4506,
+      "step": 1447
+    },
+    {
+      "epoch": 1.8021157436216553,
+      "grad_norm": 2.953301429748535,
+      "learning_rate": 2.253228782287823e-06,
+      "loss": 1.4224,
+      "step": 1448
+    },
+    {
+      "epoch": 1.8033602986932171,
+      "grad_norm": 3.0210883617401123,
+      "learning_rate": 2.2509225092250924e-06,
+      "loss": 1.4387,
+      "step": 1449
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "grad_norm": 2.975741386413574,
+      "learning_rate": 2.248616236162362e-06,
+      "loss": 1.4318,
+      "step": 1450
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "eval_loss": 1.618930697441101,
+      "eval_runtime": 49.6692,
+      "eval_samples_per_second": 20.133,
+      "eval_steps_per_second": 0.846,
+      "step": 1450
+    },
+    {
+      "epoch": 1.805849408836341,
+      "grad_norm": 2.9083876609802246,
+      "learning_rate": 2.2463099630996312e-06,
+      "loss": 1.3837,
+      "step": 1451
+    },
+    {
+      "epoch": 1.807093963907903,
+      "grad_norm": 3.364248275756836,
+      "learning_rate": 2.2440036900369006e-06,
+      "loss": 1.4589,
+      "step": 1452
+    },
+    {
+      "epoch": 1.8083385189794647,
+      "grad_norm": 2.935478925704956,
+      "learning_rate": 2.2416974169741696e-06,
+      "loss": 1.4119,
+      "step": 1453
+    },
+    {
+      "epoch": 1.8095830740510268,
+      "grad_norm": 3.1742911338806152,
+      "learning_rate": 2.2393911439114395e-06,
+      "loss": 1.4203,
+      "step": 1454
+    },
+    {
+      "epoch": 1.8108276291225887,
+      "grad_norm": 2.96966290473938,
+      "learning_rate": 2.2370848708487085e-06,
+      "loss": 1.4054,
+      "step": 1455
+    },
+    {
+      "epoch": 1.8120721841941507,
+      "grad_norm": 3.0919055938720703,
+      "learning_rate": 2.2347785977859783e-06,
+      "loss": 1.4289,
+      "step": 1456
+    },
+    {
+      "epoch": 1.8133167392657126,
+      "grad_norm": 2.9061055183410645,
+      "learning_rate": 2.2324723247232473e-06,
+      "loss": 1.4958,
+      "step": 1457
+    },
+    {
+      "epoch": 1.8145612943372744,
+      "grad_norm": 2.968348503112793,
+      "learning_rate": 2.2301660516605168e-06,
+      "loss": 1.4033,
+      "step": 1458
+    },
+    {
+      "epoch": 1.8158058494088363,
+      "grad_norm": 3.0435378551483154,
+      "learning_rate": 2.227859778597786e-06,
+      "loss": 1.414,
+      "step": 1459
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "grad_norm": 2.9067916870117188,
+      "learning_rate": 2.2255535055350556e-06,
+      "loss": 1.3954,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "eval_loss": 1.6240814924240112,
+      "eval_runtime": 51.7164,
+      "eval_samples_per_second": 19.336,
+      "eval_steps_per_second": 0.812,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8182949595519602,
+      "grad_norm": 3.156384229660034,
+      "learning_rate": 2.2232472324723246e-06,
+      "loss": 1.4658,
+      "step": 1461
+    },
+    {
+      "epoch": 1.819539514623522,
+      "grad_norm": 2.9885313510894775,
+      "learning_rate": 2.2209409594095944e-06,
+      "loss": 1.4123,
+      "step": 1462
+    },
+    {
+      "epoch": 1.820784069695084,
+      "grad_norm": 3.227780818939209,
+      "learning_rate": 2.2186346863468634e-06,
+      "loss": 1.3587,
+      "step": 1463
+    },
+    {
+      "epoch": 1.822028624766646,
+      "grad_norm": 3.0375192165374756,
+      "learning_rate": 2.2163284132841333e-06,
+      "loss": 1.4015,
+      "step": 1464
+    },
+    {
+      "epoch": 1.8232731798382078,
+      "grad_norm": 3.0200846195220947,
+      "learning_rate": 2.2140221402214023e-06,
+      "loss": 1.419,
+      "step": 1465
+    },
+    {
+      "epoch": 1.8245177349097697,
+      "grad_norm": 2.9456615447998047,
+      "learning_rate": 2.2117158671586717e-06,
+      "loss": 1.4075,
+      "step": 1466
+    },
+    {
+      "epoch": 1.8257622899813317,
+      "grad_norm": 3.3129165172576904,
+      "learning_rate": 2.209409594095941e-06,
+      "loss": 1.4241,
+      "step": 1467
+    },
+    {
+      "epoch": 1.8270068450528936,
+      "grad_norm": 3.013141393661499,
+      "learning_rate": 2.2071033210332106e-06,
+      "loss": 1.4066,
+      "step": 1468
+    },
+    {
+      "epoch": 1.8282514001244556,
+      "grad_norm": 2.8414366245269775,
+      "learning_rate": 2.20479704797048e-06,
+      "loss": 1.4009,
+      "step": 1469
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "grad_norm": 2.8920438289642334,
+      "learning_rate": 2.2024907749077494e-06,
+      "loss": 1.3916,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "eval_loss": 1.6220412254333496,
+      "eval_runtime": 52.6199,
+      "eval_samples_per_second": 19.004,
+      "eval_steps_per_second": 0.798,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8307405102675793,
+      "grad_norm": 3.032353639602661,
+      "learning_rate": 2.2001845018450184e-06,
+      "loss": 1.402,
+      "step": 1471
+    },
+    {
+      "epoch": 1.8319850653391412,
+      "grad_norm": 2.9512081146240234,
+      "learning_rate": 2.1978782287822883e-06,
+      "loss": 1.429,
+      "step": 1472
+    },
+    {
+      "epoch": 1.833229620410703,
+      "grad_norm": 2.846081018447876,
+      "learning_rate": 2.1955719557195573e-06,
+      "loss": 1.4203,
+      "step": 1473
+    },
+    {
+      "epoch": 1.834474175482265,
+      "grad_norm": 3.093378782272339,
+      "learning_rate": 2.1932656826568267e-06,
+      "loss": 1.4307,
+      "step": 1474
+    },
+    {
+      "epoch": 1.8357187305538272,
+      "grad_norm": 2.808326005935669,
+      "learning_rate": 2.190959409594096e-06,
+      "loss": 1.3753,
+      "step": 1475
+    },
+    {
+      "epoch": 1.836963285625389,
+      "grad_norm": 3.0328383445739746,
+      "learning_rate": 2.1886531365313655e-06,
+      "loss": 1.4534,
+      "step": 1476
+    },
+    {
+      "epoch": 1.8382078406969509,
+      "grad_norm": 3.045897960662842,
+      "learning_rate": 2.186346863468635e-06,
+      "loss": 1.3259,
+      "step": 1477
+    },
+    {
+      "epoch": 1.8394523957685127,
+      "grad_norm": 3.171846628189087,
+      "learning_rate": 2.1840405904059044e-06,
+      "loss": 1.397,
+      "step": 1478
+    },
+    {
+      "epoch": 1.8406969508400746,
+      "grad_norm": 2.893061637878418,
+      "learning_rate": 2.1817343173431734e-06,
+      "loss": 1.4201,
+      "step": 1479
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "grad_norm": 3.0151467323303223,
+      "learning_rate": 2.1794280442804432e-06,
+      "loss": 1.4266,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "eval_loss": 1.6166906356811523,
+      "eval_runtime": 51.8266,
+      "eval_samples_per_second": 19.295,
+      "eval_steps_per_second": 0.81,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8431860609831985,
+      "grad_norm": 2.942431688308716,
+      "learning_rate": 2.177121771217712e-06,
+      "loss": 1.3744,
+      "step": 1481
+    },
+    {
+      "epoch": 1.8444306160547606,
+      "grad_norm": 2.8482232093811035,
+      "learning_rate": 2.1748154981549816e-06,
+      "loss": 1.3914,
+      "step": 1482
+    },
+    {
+      "epoch": 1.8456751711263224,
+      "grad_norm": 2.9315359592437744,
+      "learning_rate": 2.172509225092251e-06,
+      "loss": 1.419,
+      "step": 1483
+    },
+    {
+      "epoch": 1.8469197261978842,
+      "grad_norm": 3.191800594329834,
+      "learning_rate": 2.1702029520295205e-06,
+      "loss": 1.3524,
+      "step": 1484
+    },
+    {
+      "epoch": 1.848164281269446,
+      "grad_norm": 2.9308040142059326,
+      "learning_rate": 2.16789667896679e-06,
+      "loss": 1.3742,
+      "step": 1485
+    },
+    {
+      "epoch": 1.849408836341008,
+      "grad_norm": 2.7678964138031006,
+      "learning_rate": 2.1655904059040593e-06,
+      "loss": 1.3786,
+      "step": 1486
+    },
+    {
+      "epoch": 1.85065339141257,
+      "grad_norm": 3.0300636291503906,
+      "learning_rate": 2.1632841328413287e-06,
+      "loss": 1.4469,
+      "step": 1487
+    },
+    {
+      "epoch": 1.851897946484132,
+      "grad_norm": 3.142781972885132,
+      "learning_rate": 2.160977859778598e-06,
+      "loss": 1.4565,
+      "step": 1488
+    },
+    {
+      "epoch": 1.853142501555694,
+      "grad_norm": 2.954916000366211,
+      "learning_rate": 2.158671586715867e-06,
+      "loss": 1.4356,
+      "step": 1489
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "grad_norm": 2.864875316619873,
+      "learning_rate": 2.1563653136531366e-06,
+      "loss": 1.4521,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "eval_loss": 1.6207302808761597,
+      "eval_runtime": 49.2215,
+      "eval_samples_per_second": 20.316,
+      "eval_steps_per_second": 0.853,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8556316116988176,
+      "grad_norm": 2.8908467292785645,
+      "learning_rate": 2.154059040590406e-06,
+      "loss": 1.4304,
+      "step": 1491
+    },
+    {
+      "epoch": 1.8568761667703795,
+      "grad_norm": 2.9439549446105957,
+      "learning_rate": 2.1517527675276754e-06,
+      "loss": 1.398,
+      "step": 1492
+    },
+    {
+      "epoch": 1.8581207218419415,
+      "grad_norm": 2.967728853225708,
+      "learning_rate": 2.149446494464945e-06,
+      "loss": 1.5038,
+      "step": 1493
+    },
+    {
+      "epoch": 1.8593652769135034,
+      "grad_norm": 2.938123941421509,
+      "learning_rate": 2.1471402214022143e-06,
+      "loss": 1.4395,
+      "step": 1494
+    },
+    {
+      "epoch": 1.8606098319850655,
+      "grad_norm": 2.93609881401062,
+      "learning_rate": 2.1448339483394837e-06,
+      "loss": 1.3939,
+      "step": 1495
+    },
+    {
+      "epoch": 1.8618543870566273,
+      "grad_norm": 2.9093775749206543,
+      "learning_rate": 2.142527675276753e-06,
+      "loss": 1.3942,
+      "step": 1496
+    },
+    {
+      "epoch": 1.8630989421281892,
+      "grad_norm": 2.773960590362549,
+      "learning_rate": 2.140221402214022e-06,
+      "loss": 1.3886,
+      "step": 1497
+    },
+    {
+      "epoch": 1.864343497199751,
+      "grad_norm": 2.9858205318450928,
+      "learning_rate": 2.1379151291512916e-06,
+      "loss": 1.3814,
+      "step": 1498
+    },
+    {
+      "epoch": 1.8655880522713129,
+      "grad_norm": 3.1233417987823486,
+      "learning_rate": 2.135608856088561e-06,
+      "loss": 1.4313,
+      "step": 1499
+    },
+    {
+      "epoch": 1.866832607342875,
+      "grad_norm": 2.9569480419158936,
+      "learning_rate": 2.1333025830258304e-06,
+      "loss": 1.4302,
+      "step": 1500
+    },
+    {
+      "epoch": 1.866832607342875,
+      "eval_loss": 1.6130231618881226,
+      "eval_runtime": 46.2834,
+      "eval_samples_per_second": 21.606,
+      "eval_steps_per_second": 0.907,
+      "step": 1500
+    },
+    {
+      "epoch": 1.868077162414437,
+      "grad_norm": 2.926203727722168,
+      "learning_rate": 2.1309963099631e-06,
+      "loss": 1.4236,
+      "step": 1501
+    },
+    {
+      "epoch": 1.8693217174859988,
+      "grad_norm": 2.9496357440948486,
+      "learning_rate": 2.1286900369003692e-06,
+      "loss": 1.4014,
+      "step": 1502
+    },
+    {
+      "epoch": 1.8705662725575607,
+      "grad_norm": 2.9314279556274414,
+      "learning_rate": 2.1263837638376387e-06,
+      "loss": 1.4414,
+      "step": 1503
+    },
+    {
+      "epoch": 1.8718108276291225,
+      "grad_norm": 2.889460802078247,
+      "learning_rate": 2.124077490774908e-06,
+      "loss": 1.3883,
+      "step": 1504
+    },
+    {
+      "epoch": 1.8730553827006844,
+      "grad_norm": 2.9859671592712402,
+      "learning_rate": 2.1217712177121775e-06,
+      "loss": 1.4551,
+      "step": 1505
+    },
+    {
+      "epoch": 1.8742999377722465,
+      "grad_norm": 2.9330570697784424,
+      "learning_rate": 2.1194649446494465e-06,
+      "loss": 1.3895,
+      "step": 1506
+    },
+    {
+      "epoch": 1.8755444928438083,
+      "grad_norm": 2.9062387943267822,
+      "learning_rate": 2.117158671586716e-06,
+      "loss": 1.4492,
+      "step": 1507
+    },
+    {
+      "epoch": 1.8767890479153704,
+      "grad_norm": 3.0337791442871094,
+      "learning_rate": 2.1148523985239854e-06,
+      "loss": 1.4682,
+      "step": 1508
+    },
+    {
+      "epoch": 1.8780336029869322,
+      "grad_norm": 3.0448431968688965,
+      "learning_rate": 2.1125461254612548e-06,
+      "loss": 1.4836,
+      "step": 1509
+    },
+    {
+      "epoch": 1.879278158058494,
+      "grad_norm": 2.912764310836792,
+      "learning_rate": 2.110239852398524e-06,
+      "loss": 1.4644,
+      "step": 1510
+    },
+    {
+      "epoch": 1.879278158058494,
+      "eval_loss": 1.6124104261398315,
+      "eval_runtime": 41.9713,
+      "eval_samples_per_second": 23.826,
+      "eval_steps_per_second": 1.001,
+      "step": 1510
+    },
+    {
+      "epoch": 1.880522713130056,
+      "grad_norm": 3.0399203300476074,
+      "learning_rate": 2.1079335793357936e-06,
+      "loss": 1.4391,
+      "step": 1511
+    },
+    {
+      "epoch": 1.8817672682016178,
+      "grad_norm": 2.9852235317230225,
+      "learning_rate": 2.1056273062730626e-06,
+      "loss": 1.346,
+      "step": 1512
+    },
+    {
+      "epoch": 1.8830118232731798,
+      "grad_norm": 2.904437780380249,
+      "learning_rate": 2.1033210332103325e-06,
+      "loss": 1.4523,
+      "step": 1513
+    },
+    {
+      "epoch": 1.884256378344742,
+      "grad_norm": 2.8988049030303955,
+      "learning_rate": 2.1010147601476015e-06,
+      "loss": 1.4001,
+      "step": 1514
+    },
+    {
+      "epoch": 1.8855009334163038,
+      "grad_norm": 2.9890260696411133,
+      "learning_rate": 2.098708487084871e-06,
+      "loss": 1.4074,
+      "step": 1515
+    },
+    {
+      "epoch": 1.8867454884878656,
+      "grad_norm": 3.0486819744110107,
+      "learning_rate": 2.0964022140221403e-06,
+      "loss": 1.461,
+      "step": 1516
+    },
+    {
+      "epoch": 1.8879900435594275,
+      "grad_norm": 2.8746016025543213,
+      "learning_rate": 2.0940959409594097e-06,
+      "loss": 1.4013,
+      "step": 1517
+    },
+    {
+      "epoch": 1.8892345986309893,
+      "grad_norm": 2.979480266571045,
+      "learning_rate": 2.091789667896679e-06,
+      "loss": 1.4567,
+      "step": 1518
+    },
+    {
+      "epoch": 1.8904791537025514,
+      "grad_norm": 2.8351497650146484,
+      "learning_rate": 2.0894833948339486e-06,
+      "loss": 1.3655,
+      "step": 1519
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "grad_norm": 2.9061646461486816,
+      "learning_rate": 2.0871771217712176e-06,
+      "loss": 1.4165,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "eval_loss": 1.6172568798065186,
+      "eval_runtime": 42.998,
+      "eval_samples_per_second": 23.257,
+      "eval_steps_per_second": 0.977,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8929682638456753,
+      "grad_norm": 2.8751838207244873,
+      "learning_rate": 2.0848708487084874e-06,
+      "loss": 1.4133,
+      "step": 1521
+    },
+    {
+      "epoch": 1.8942128189172371,
+      "grad_norm": 2.7513587474823,
+      "learning_rate": 2.0825645756457564e-06,
+      "loss": 1.3734,
+      "step": 1522
+    },
+    {
+      "epoch": 1.895457373988799,
+      "grad_norm": 2.9244418144226074,
+      "learning_rate": 2.0802583025830263e-06,
+      "loss": 1.4181,
+      "step": 1523
+    },
+    {
+      "epoch": 1.8967019290603608,
+      "grad_norm": 2.877495765686035,
+      "learning_rate": 2.0779520295202953e-06,
+      "loss": 1.4009,
+      "step": 1524
+    },
+    {
+      "epoch": 1.8979464841319227,
+      "grad_norm": 2.9316184520721436,
+      "learning_rate": 2.0756457564575647e-06,
+      "loss": 1.3796,
+      "step": 1525
+    },
+    {
+      "epoch": 1.8991910392034848,
+      "grad_norm": 3.0403976440429688,
+      "learning_rate": 2.073339483394834e-06,
+      "loss": 1.4101,
+      "step": 1526
+    },
+    {
+      "epoch": 1.9004355942750468,
+      "grad_norm": 3.0020930767059326,
+      "learning_rate": 2.0710332103321035e-06,
+      "loss": 1.4166,
+      "step": 1527
+    },
+    {
+      "epoch": 1.9016801493466087,
+      "grad_norm": 3.0937304496765137,
+      "learning_rate": 2.0687269372693725e-06,
+      "loss": 1.4599,
+      "step": 1528
+    },
+    {
+      "epoch": 1.9029247044181705,
+      "grad_norm": 2.901540517807007,
+      "learning_rate": 2.0664206642066424e-06,
+      "loss": 1.3642,
+      "step": 1529
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "grad_norm": 3.0517938137054443,
+      "learning_rate": 2.0641143911439114e-06,
+      "loss": 1.451,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "eval_loss": 1.6178728342056274,
+      "eval_runtime": 43.9067,
+      "eval_samples_per_second": 22.776,
+      "eval_steps_per_second": 0.957,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9054138145612942,
+      "grad_norm": 3.12484073638916,
+      "learning_rate": 2.0618081180811812e-06,
+      "loss": 1.4522,
+      "step": 1531
+    },
+    {
+      "epoch": 1.9066583696328563,
+      "grad_norm": 2.9447414875030518,
+      "learning_rate": 2.0595018450184502e-06,
+      "loss": 1.4154,
+      "step": 1532
+    },
+    {
+      "epoch": 1.9079029247044181,
+      "grad_norm": 2.909738779067993,
+      "learning_rate": 2.0571955719557197e-06,
+      "loss": 1.4133,
+      "step": 1533
+    },
+    {
+      "epoch": 1.9091474797759802,
+      "grad_norm": 3.1522421836853027,
+      "learning_rate": 2.054889298892989e-06,
+      "loss": 1.4777,
+      "step": 1534
+    },
+    {
+      "epoch": 1.910392034847542,
+      "grad_norm": 3.041447639465332,
+      "learning_rate": 2.0525830258302585e-06,
+      "loss": 1.4104,
+      "step": 1535
+    },
+    {
+      "epoch": 1.911636589919104,
+      "grad_norm": 3.01129412651062,
+      "learning_rate": 2.050276752767528e-06,
+      "loss": 1.3852,
+      "step": 1536
+    },
+    {
+      "epoch": 1.9128811449906657,
+      "grad_norm": 3.0697543621063232,
+      "learning_rate": 2.0479704797047974e-06,
+      "loss": 1.4389,
+      "step": 1537
+    },
+    {
+      "epoch": 1.9141257000622276,
+      "grad_norm": 2.9391236305236816,
+      "learning_rate": 2.0456642066420664e-06,
+      "loss": 1.4239,
+      "step": 1538
+    },
+    {
+      "epoch": 1.9153702551337897,
+      "grad_norm": 2.978236675262451,
+      "learning_rate": 2.043357933579336e-06,
+      "loss": 1.462,
+      "step": 1539
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "grad_norm": 2.8124167919158936,
+      "learning_rate": 2.041051660516605e-06,
+      "loss": 1.3186,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "eval_loss": 1.6120468378067017,
+      "eval_runtime": 43.3787,
+      "eval_samples_per_second": 23.053,
+      "eval_steps_per_second": 0.968,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9178593652769136,
+      "grad_norm": 3.029939889907837,
+      "learning_rate": 2.0387453874538746e-06,
+      "loss": 1.4481,
+      "step": 1541
+    },
+    {
+      "epoch": 1.9191039203484754,
+      "grad_norm": 2.8749356269836426,
+      "learning_rate": 2.036439114391144e-06,
+      "loss": 1.3735,
+      "step": 1542
+    },
+    {
+      "epoch": 1.9203484754200373,
+      "grad_norm": 2.9181101322174072,
+      "learning_rate": 2.0341328413284135e-06,
+      "loss": 1.4085,
+      "step": 1543
+    },
+    {
+      "epoch": 1.9215930304915991,
+      "grad_norm": 2.936319589614868,
+      "learning_rate": 2.031826568265683e-06,
+      "loss": 1.3827,
+      "step": 1544
+    },
+    {
+      "epoch": 1.9228375855631612,
+      "grad_norm": 2.9286844730377197,
+      "learning_rate": 2.0295202952029523e-06,
+      "loss": 1.4563,
+      "step": 1545
+    },
+    {
+      "epoch": 1.924082140634723,
+      "grad_norm": 2.790752649307251,
+      "learning_rate": 2.0272140221402213e-06,
+      "loss": 1.3935,
+      "step": 1546
+    },
+    {
+      "epoch": 1.9253266957062851,
+      "grad_norm": 2.878164529800415,
+      "learning_rate": 2.024907749077491e-06,
+      "loss": 1.4159,
+      "step": 1547
+    },
+    {
+      "epoch": 1.926571250777847,
+      "grad_norm": 3.0583767890930176,
+      "learning_rate": 2.02260147601476e-06,
+      "loss": 1.4036,
+      "step": 1548
+    },
+    {
+      "epoch": 1.9278158058494088,
+      "grad_norm": 2.943686008453369,
+      "learning_rate": 2.02029520295203e-06,
+      "loss": 1.4178,
+      "step": 1549
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "grad_norm": 2.983553886413574,
+      "learning_rate": 2.017988929889299e-06,
+      "loss": 1.3959,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "eval_loss": 1.610864520072937,
+      "eval_runtime": 44.5262,
+      "eval_samples_per_second": 22.459,
+      "eval_steps_per_second": 0.943,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9303049159925325,
+      "grad_norm": 3.0323703289031982,
+      "learning_rate": 2.0156826568265684e-06,
+      "loss": 1.4362,
+      "step": 1551
+    },
+    {
+      "epoch": 1.9315494710640946,
+      "grad_norm": 3.036627769470215,
+      "learning_rate": 2.013376383763838e-06,
+      "loss": 1.4132,
+      "step": 1552
+    },
+    {
+      "epoch": 1.9327940261356567,
+      "grad_norm": 2.901071310043335,
+      "learning_rate": 2.0110701107011073e-06,
+      "loss": 1.3846,
+      "step": 1553
+    },
+    {
+      "epoch": 1.9340385812072185,
+      "grad_norm": 3.002803325653076,
+      "learning_rate": 2.0087638376383767e-06,
+      "loss": 1.4402,
+      "step": 1554
+    },
+    {
+      "epoch": 1.9352831362787803,
+      "grad_norm": 2.969228982925415,
+      "learning_rate": 2.006457564575646e-06,
+      "loss": 1.4387,
+      "step": 1555
+    },
+    {
+      "epoch": 1.9365276913503422,
+      "grad_norm": 2.9773998260498047,
+      "learning_rate": 2.004151291512915e-06,
+      "loss": 1.4149,
+      "step": 1556
+    },
+    {
+      "epoch": 1.937772246421904,
+      "grad_norm": 2.831655502319336,
+      "learning_rate": 2.0018450184501845e-06,
+      "loss": 1.3753,
+      "step": 1557
+    },
+    {
+      "epoch": 1.9390168014934661,
+      "grad_norm": 2.835994005203247,
+      "learning_rate": 1.999538745387454e-06,
+      "loss": 1.345,
+      "step": 1558
+    },
+    {
+      "epoch": 1.940261356565028,
+      "grad_norm": 3.0328211784362793,
+      "learning_rate": 1.9972324723247234e-06,
+      "loss": 1.3473,
+      "step": 1559
+    },
+    {
+      "epoch": 1.94150591163659,
+      "grad_norm": 2.9441378116607666,
+      "learning_rate": 1.994926199261993e-06,
+      "loss": 1.3913,
+      "step": 1560
+    },
+    {
+      "epoch": 1.94150591163659,
+      "eval_loss": 1.6193854808807373,
+      "eval_runtime": 44.6194,
+      "eval_samples_per_second": 22.412,
+      "eval_steps_per_second": 0.941,
+      "step": 1560
+    },
+    {
+      "epoch": 1.9427504667081519,
+      "grad_norm": 2.905486822128296,
+      "learning_rate": 1.9926199261992622e-06,
+      "loss": 1.4043,
+      "step": 1561
+    },
+    {
+      "epoch": 1.9439950217797137,
+      "grad_norm": 2.7758026123046875,
+      "learning_rate": 1.9903136531365317e-06,
+      "loss": 1.3974,
+      "step": 1562
+    },
+    {
+      "epoch": 1.9452395768512756,
+      "grad_norm": 2.7883355617523193,
+      "learning_rate": 1.988007380073801e-06,
+      "loss": 1.4246,
+      "step": 1563
+    },
+    {
+      "epoch": 1.9464841319228374,
+      "grad_norm": 2.8322808742523193,
+      "learning_rate": 1.98570110701107e-06,
+      "loss": 1.4002,
+      "step": 1564
+    },
+    {
+      "epoch": 1.9477286869943995,
+      "grad_norm": 2.948850631713867,
+      "learning_rate": 1.9833948339483395e-06,
+      "loss": 1.3866,
+      "step": 1565
+    },
+    {
+      "epoch": 1.9489732420659616,
+      "grad_norm": 2.9772586822509766,
+      "learning_rate": 1.981088560885609e-06,
+      "loss": 1.4473,
+      "step": 1566
+    },
+    {
+      "epoch": 1.9502177971375234,
+      "grad_norm": 2.9183740615844727,
+      "learning_rate": 1.9787822878228783e-06,
+      "loss": 1.3943,
+      "step": 1567
+    },
+    {
+      "epoch": 1.9514623522090853,
+      "grad_norm": 3.10422682762146,
+      "learning_rate": 1.9764760147601478e-06,
+      "loss": 1.465,
+      "step": 1568
+    },
+    {
+      "epoch": 1.952706907280647,
+      "grad_norm": 3.001533031463623,
+      "learning_rate": 1.974169741697417e-06,
+      "loss": 1.4133,
+      "step": 1569
+    },
+    {
+      "epoch": 1.953951462352209,
+      "grad_norm": 2.9521870613098145,
+      "learning_rate": 1.9718634686346866e-06,
+      "loss": 1.4652,
+      "step": 1570
+    },
+    {
+      "epoch": 1.953951462352209,
+      "eval_loss": 1.617977499961853,
+      "eval_runtime": 45.7189,
+      "eval_samples_per_second": 21.873,
+      "eval_steps_per_second": 0.919,
+      "step": 1570
+    },
+    {
+      "epoch": 1.955196017423771,
+      "grad_norm": 2.9504826068878174,
+      "learning_rate": 1.969557195571956e-06,
+      "loss": 1.4117,
+      "step": 1571
+    },
+    {
+      "epoch": 1.9564405724953329,
+      "grad_norm": 2.9383559226989746,
+      "learning_rate": 1.9672509225092255e-06,
+      "loss": 1.3439,
+      "step": 1572
+    },
+    {
+      "epoch": 1.957685127566895,
+      "grad_norm": 2.850644588470459,
+      "learning_rate": 1.9649446494464945e-06,
+      "loss": 1.3521,
+      "step": 1573
+    },
+    {
+      "epoch": 1.9589296826384568,
+      "grad_norm": 2.8787410259246826,
+      "learning_rate": 1.962638376383764e-06,
+      "loss": 1.4268,
+      "step": 1574
+    },
+    {
+      "epoch": 1.9601742377100186,
+      "grad_norm": 2.966080665588379,
+      "learning_rate": 1.9603321033210333e-06,
+      "loss": 1.3367,
+      "step": 1575
+    },
+    {
+      "epoch": 1.9614187927815805,
+      "grad_norm": 2.858222007751465,
+      "learning_rate": 1.9580258302583027e-06,
+      "loss": 1.3842,
+      "step": 1576
+    },
+    {
+      "epoch": 1.9626633478531423,
+      "grad_norm": 2.924525737762451,
+      "learning_rate": 1.955719557195572e-06,
+      "loss": 1.4561,
+      "step": 1577
+    },
+    {
+      "epoch": 1.9639079029247044,
+      "grad_norm": 2.9281182289123535,
+      "learning_rate": 1.9534132841328416e-06,
+      "loss": 1.4179,
+      "step": 1578
+    },
+    {
+      "epoch": 1.9651524579962665,
+      "grad_norm": 2.982023000717163,
+      "learning_rate": 1.951107011070111e-06,
+      "loss": 1.4413,
+      "step": 1579
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "grad_norm": 2.7912652492523193,
+      "learning_rate": 1.9488007380073804e-06,
+      "loss": 1.3577,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "eval_loss": 1.6062211990356445,
+      "eval_runtime": 46.7931,
+      "eval_samples_per_second": 21.371,
+      "eval_steps_per_second": 0.898,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9676415681393902,
+      "grad_norm": 2.9585583209991455,
+      "learning_rate": 1.9464944649446494e-06,
+      "loss": 1.4084,
+      "step": 1581
+    },
+    {
+      "epoch": 1.968886123210952,
+      "grad_norm": 2.879201889038086,
+      "learning_rate": 1.944188191881919e-06,
+      "loss": 1.4164,
+      "step": 1582
+    },
+    {
+      "epoch": 1.9701306782825139,
+      "grad_norm": 2.9510116577148438,
+      "learning_rate": 1.9418819188191883e-06,
+      "loss": 1.4211,
+      "step": 1583
+    },
+    {
+      "epoch": 1.971375233354076,
+      "grad_norm": 2.973844528198242,
+      "learning_rate": 1.9395756457564577e-06,
+      "loss": 1.4358,
+      "step": 1584
+    },
+    {
+      "epoch": 1.9726197884256378,
+      "grad_norm": 2.939131736755371,
+      "learning_rate": 1.937269372693727e-06,
+      "loss": 1.407,
+      "step": 1585
+    },
+    {
+      "epoch": 1.9738643434971999,
+      "grad_norm": 2.956646680831909,
+      "learning_rate": 1.9349630996309965e-06,
+      "loss": 1.3872,
+      "step": 1586
+    },
+    {
+      "epoch": 1.9751088985687617,
+      "grad_norm": 3.070354461669922,
+      "learning_rate": 1.932656826568266e-06,
+      "loss": 1.412,
+      "step": 1587
+    },
+    {
+      "epoch": 1.9763534536403236,
+      "grad_norm": 2.8143749237060547,
+      "learning_rate": 1.9303505535055354e-06,
+      "loss": 1.3522,
+      "step": 1588
+    },
+    {
+      "epoch": 1.9775980087118854,
+      "grad_norm": 2.8610002994537354,
+      "learning_rate": 1.9280442804428044e-06,
+      "loss": 1.3988,
+      "step": 1589
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "grad_norm": 2.951023817062378,
+      "learning_rate": 1.925738007380074e-06,
+      "loss": 1.3855,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "eval_loss": 1.613527536392212,
+      "eval_runtime": 46.5242,
+      "eval_samples_per_second": 21.494,
+      "eval_steps_per_second": 0.903,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9800871188550093,
+      "grad_norm": 2.9333510398864746,
+      "learning_rate": 1.9234317343173432e-06,
+      "loss": 1.4518,
+      "step": 1591
+    },
+    {
+      "epoch": 1.9813316739265714,
+      "grad_norm": 2.847015619277954,
+      "learning_rate": 1.9211254612546126e-06,
+      "loss": 1.3999,
+      "step": 1592
+    },
+    {
+      "epoch": 1.9825762289981332,
+      "grad_norm": 2.969320058822632,
+      "learning_rate": 1.918819188191882e-06,
+      "loss": 1.4241,
+      "step": 1593
+    },
+    {
+      "epoch": 1.983820784069695,
+      "grad_norm": 2.862797498703003,
+      "learning_rate": 1.9165129151291515e-06,
+      "loss": 1.3929,
+      "step": 1594
+    },
+    {
+      "epoch": 1.985065339141257,
+      "grad_norm": 2.90384840965271,
+      "learning_rate": 1.914206642066421e-06,
+      "loss": 1.4127,
+      "step": 1595
+    },
+    {
+      "epoch": 1.9863098942128188,
+      "grad_norm": 2.8059256076812744,
+      "learning_rate": 1.9119003690036903e-06,
+      "loss": 1.4155,
+      "step": 1596
+    },
+    {
+      "epoch": 1.9875544492843809,
+      "grad_norm": 2.9068455696105957,
+      "learning_rate": 1.9095940959409593e-06,
+      "loss": 1.3924,
+      "step": 1597
+    },
+    {
+      "epoch": 1.9887990043559427,
+      "grad_norm": 2.903542995452881,
+      "learning_rate": 1.907287822878229e-06,
+      "loss": 1.4039,
+      "step": 1598
+    },
+    {
+      "epoch": 1.9900435594275048,
+      "grad_norm": 2.8566174507141113,
+      "learning_rate": 1.9049815498154984e-06,
+      "loss": 1.4276,
+      "step": 1599
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "grad_norm": 2.8891515731811523,
+      "learning_rate": 1.9026752767527676e-06,
+      "loss": 1.4555,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "eval_loss": 1.6086686849594116,
+      "eval_runtime": 46.2785,
+      "eval_samples_per_second": 21.608,
+      "eval_steps_per_second": 0.908,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9925326695706285,
+      "grad_norm": 2.933851957321167,
+      "learning_rate": 1.900369003690037e-06,
+      "loss": 1.3881,
+      "step": 1601
+    },
+    {
+      "epoch": 1.9937772246421903,
+      "grad_norm": 2.8393521308898926,
+      "learning_rate": 1.8980627306273065e-06,
+      "loss": 1.3959,
+      "step": 1602
+    },
+    {
+      "epoch": 1.9950217797137524,
+      "grad_norm": 2.839740753173828,
+      "learning_rate": 1.8957564575645759e-06,
+      "loss": 1.384,
+      "step": 1603
+    },
+    {
+      "epoch": 1.9962663347853142,
+      "grad_norm": 2.7918097972869873,
+      "learning_rate": 1.8934501845018453e-06,
+      "loss": 1.4173,
+      "step": 1604
+    },
+    {
+      "epoch": 1.9975108898568763,
+      "grad_norm": 2.920839548110962,
+      "learning_rate": 1.8911439114391145e-06,
+      "loss": 1.3693,
+      "step": 1605
+    },
+    {
+      "epoch": 1.9987554449284382,
+      "grad_norm": 2.9545562267303467,
+      "learning_rate": 1.888837638376384e-06,
+      "loss": 1.4062,
+      "step": 1606
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 2.912815570831299,
+      "learning_rate": 1.8865313653136534e-06,
+      "loss": 1.4146,
+      "step": 1607
+    },
+    {
+      "epoch": 2.001244555071562,
+      "grad_norm": 3.1125903129577637,
+      "learning_rate": 1.8842250922509226e-06,
+      "loss": 1.3294,
+      "step": 1608
+    },
+    {
+      "epoch": 2.0024891101431237,
+      "grad_norm": 3.123905658721924,
+      "learning_rate": 1.881918819188192e-06,
+      "loss": 1.4175,
+      "step": 1609
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "grad_norm": 3.055100440979004,
+      "learning_rate": 1.8796125461254614e-06,
+      "loss": 1.3628,
+      "step": 1610
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "eval_loss": 1.6198724508285522,
+      "eval_runtime": 42.5155,
+      "eval_samples_per_second": 23.521,
+      "eval_steps_per_second": 0.988,
+      "step": 1610
+    },
+    {
+      "epoch": 2.004978220286248,
+      "grad_norm": 3.087313413619995,
+      "learning_rate": 1.8773062730627308e-06,
+      "loss": 1.292,
+      "step": 1611
+    },
+    {
+      "epoch": 2.0062227753578097,
+      "grad_norm": 3.1849849224090576,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 1.3393,
+      "step": 1612
+    },
+    {
+      "epoch": 2.0074673304293715,
+      "grad_norm": 3.243750810623169,
+      "learning_rate": 1.8726937269372695e-06,
+      "loss": 1.3367,
+      "step": 1613
+    },
+    {
+      "epoch": 2.0087118855009334,
+      "grad_norm": 3.3086085319519043,
+      "learning_rate": 1.8703874538745389e-06,
+      "loss": 1.3099,
+      "step": 1614
+    },
+    {
+      "epoch": 2.0099564405724952,
+      "grad_norm": 3.388633966445923,
+      "learning_rate": 1.8680811808118083e-06,
+      "loss": 1.3816,
+      "step": 1615
+    },
+    {
+      "epoch": 2.011200995644057,
+      "grad_norm": 3.422671318054199,
+      "learning_rate": 1.8657749077490777e-06,
+      "loss": 1.3799,
+      "step": 1616
+    },
+    {
+      "epoch": 2.0124455507156194,
+      "grad_norm": 3.1104466915130615,
+      "learning_rate": 1.863468634686347e-06,
+      "loss": 1.3752,
+      "step": 1617
+    },
+    {
+      "epoch": 2.013690105787181,
+      "grad_norm": 3.2728488445281982,
+      "learning_rate": 1.8611623616236164e-06,
+      "loss": 1.322,
+      "step": 1618
+    },
+    {
+      "epoch": 2.014934660858743,
+      "grad_norm": 3.3357644081115723,
+      "learning_rate": 1.8588560885608858e-06,
+      "loss": 1.3193,
+      "step": 1619
+    },
+    {
+      "epoch": 2.016179215930305,
+      "grad_norm": 3.1026456356048584,
+      "learning_rate": 1.856549815498155e-06,
+      "loss": 1.3394,
+      "step": 1620
+    },
+    {
+      "epoch": 2.016179215930305,
+      "eval_loss": 1.6217585802078247,
+      "eval_runtime": 45.9794,
+      "eval_samples_per_second": 21.749,
+      "eval_steps_per_second": 0.913,
+      "step": 1620
+    },
+    {
+      "epoch": 2.0174237710018668,
+      "grad_norm": 3.314291000366211,
+      "learning_rate": 1.8542435424354244e-06,
+      "loss": 1.3778,
+      "step": 1621
+    },
+    {
+      "epoch": 2.0186683260734286,
+      "grad_norm": 3.3624770641326904,
+      "learning_rate": 1.8519372693726939e-06,
+      "loss": 1.3948,
+      "step": 1622
+    },
+    {
+      "epoch": 2.0199128811449905,
+      "grad_norm": 3.4041574001312256,
+      "learning_rate": 1.8496309963099633e-06,
+      "loss": 1.3109,
+      "step": 1623
+    },
+    {
+      "epoch": 2.0211574362165527,
+      "grad_norm": 3.5128726959228516,
+      "learning_rate": 1.8473247232472327e-06,
+      "loss": 1.4223,
+      "step": 1624
+    },
+    {
+      "epoch": 2.0224019912881146,
+      "grad_norm": 3.444768190383911,
+      "learning_rate": 1.845018450184502e-06,
+      "loss": 1.3672,
+      "step": 1625
+    },
+    {
+      "epoch": 2.0236465463596764,
+      "grad_norm": 3.478060722351074,
+      "learning_rate": 1.8427121771217713e-06,
+      "loss": 1.4384,
+      "step": 1626
+    },
+    {
+      "epoch": 2.0248911014312383,
+      "grad_norm": 3.211465835571289,
+      "learning_rate": 1.8404059040590408e-06,
+      "loss": 1.3891,
+      "step": 1627
+    },
+    {
+      "epoch": 2.0261356565028,
+      "grad_norm": 3.1283977031707764,
+      "learning_rate": 1.83809963099631e-06,
+      "loss": 1.3752,
+      "step": 1628
+    },
+    {
+      "epoch": 2.027380211574362,
+      "grad_norm": 3.3699934482574463,
+      "learning_rate": 1.8357933579335796e-06,
+      "loss": 1.3751,
+      "step": 1629
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "grad_norm": 3.0729823112487793,
+      "learning_rate": 1.8334870848708488e-06,
+      "loss": 1.3452,
+      "step": 1630
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "eval_loss": 1.6185853481292725,
+      "eval_runtime": 45.2873,
+      "eval_samples_per_second": 22.081,
+      "eval_steps_per_second": 0.927,
+      "step": 1630
+    },
+    {
+      "epoch": 2.029869321717486,
+      "grad_norm": 3.274139642715454,
+      "learning_rate": 1.8311808118081182e-06,
+      "loss": 1.3242,
+      "step": 1631
+    },
+    {
+      "epoch": 2.031113876789048,
+      "grad_norm": 3.2869648933410645,
+      "learning_rate": 1.8288745387453877e-06,
+      "loss": 1.3435,
+      "step": 1632
+    },
+    {
+      "epoch": 2.03235843186061,
+      "grad_norm": 3.3614306449890137,
+      "learning_rate": 1.8265682656826569e-06,
+      "loss": 1.3705,
+      "step": 1633
+    },
+    {
+      "epoch": 2.0336029869321717,
+      "grad_norm": 3.411914587020874,
+      "learning_rate": 1.8242619926199265e-06,
+      "loss": 1.3942,
+      "step": 1634
+    },
+    {
+      "epoch": 2.0348475420037335,
+      "grad_norm": 3.274519920349121,
+      "learning_rate": 1.8219557195571957e-06,
+      "loss": 1.3393,
+      "step": 1635
+    },
+    {
+      "epoch": 2.0360920970752954,
+      "grad_norm": 3.993478775024414,
+      "learning_rate": 1.819649446494465e-06,
+      "loss": 1.3713,
+      "step": 1636
+    },
+    {
+      "epoch": 2.0373366521468577,
+      "grad_norm": 3.273571491241455,
+      "learning_rate": 1.8173431734317346e-06,
+      "loss": 1.3337,
+      "step": 1637
+    },
+    {
+      "epoch": 2.0385812072184195,
+      "grad_norm": 3.1542446613311768,
+      "learning_rate": 1.8150369003690038e-06,
+      "loss": 1.3793,
+      "step": 1638
+    },
+    {
+      "epoch": 2.0398257622899814,
+      "grad_norm": 3.263773202896118,
+      "learning_rate": 1.812730627306273e-06,
+      "loss": 1.3844,
+      "step": 1639
+    },
+    {
+      "epoch": 2.041070317361543,
+      "grad_norm": 3.268772840499878,
+      "learning_rate": 1.8104243542435426e-06,
+      "loss": 1.3947,
+      "step": 1640
+    },
+    {
+      "epoch": 2.041070317361543,
+      "eval_loss": 1.6149734258651733,
+      "eval_runtime": 48.5557,
+      "eval_samples_per_second": 20.595,
+      "eval_steps_per_second": 0.865,
+      "step": 1640
+    },
+    {
+      "epoch": 2.042314872433105,
+      "grad_norm": 3.190168619155884,
+      "learning_rate": 1.8081180811808118e-06,
+      "loss": 1.3276,
+      "step": 1641
+    },
+    {
+      "epoch": 2.043559427504667,
+      "grad_norm": 3.1573212146759033,
+      "learning_rate": 1.8058118081180815e-06,
+      "loss": 1.3307,
+      "step": 1642
+    },
+    {
+      "epoch": 2.044803982576229,
+      "grad_norm": 3.286513328552246,
+      "learning_rate": 1.8035055350553507e-06,
+      "loss": 1.3359,
+      "step": 1643
+    },
+    {
+      "epoch": 2.046048537647791,
+      "grad_norm": 3.342024803161621,
+      "learning_rate": 1.8011992619926199e-06,
+      "loss": 1.3472,
+      "step": 1644
+    },
+    {
+      "epoch": 2.047293092719353,
+      "grad_norm": 3.3237056732177734,
+      "learning_rate": 1.7988929889298895e-06,
+      "loss": 1.3898,
+      "step": 1645
+    },
+    {
+      "epoch": 2.0485376477909147,
+      "grad_norm": 3.370035409927368,
+      "learning_rate": 1.7965867158671587e-06,
+      "loss": 1.3537,
+      "step": 1646
+    },
+    {
+      "epoch": 2.0497822028624766,
+      "grad_norm": 3.1636598110198975,
+      "learning_rate": 1.7942804428044284e-06,
+      "loss": 1.3455,
+      "step": 1647
+    },
+    {
+      "epoch": 2.0510267579340384,
+      "grad_norm": 3.239396810531616,
+      "learning_rate": 1.7919741697416976e-06,
+      "loss": 1.3212,
+      "step": 1648
+    },
+    {
+      "epoch": 2.0522713130056003,
+      "grad_norm": 3.2765328884124756,
+      "learning_rate": 1.7896678966789668e-06,
+      "loss": 1.396,
+      "step": 1649
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "grad_norm": 3.2259459495544434,
+      "learning_rate": 1.7873616236162364e-06,
+      "loss": 1.3614,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "eval_loss": 1.614806056022644,
+      "eval_runtime": 49.8704,
+      "eval_samples_per_second": 20.052,
+      "eval_steps_per_second": 0.842,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0547604231487244,
+      "grad_norm": 3.257683277130127,
+      "learning_rate": 1.7850553505535056e-06,
+      "loss": 1.3663,
+      "step": 1651
+    },
+    {
+      "epoch": 2.0560049782202863,
+      "grad_norm": 3.292070150375366,
+      "learning_rate": 1.7827490774907748e-06,
+      "loss": 1.3548,
+      "step": 1652
+    },
+    {
+      "epoch": 2.057249533291848,
+      "grad_norm": 3.3517003059387207,
+      "learning_rate": 1.7804428044280445e-06,
+      "loss": 1.4015,
+      "step": 1653
+    },
+    {
+      "epoch": 2.05849408836341,
+      "grad_norm": 3.5100908279418945,
+      "learning_rate": 1.7781365313653137e-06,
+      "loss": 1.3666,
+      "step": 1654
+    },
+    {
+      "epoch": 2.059738643434972,
+      "grad_norm": 3.3054633140563965,
+      "learning_rate": 1.7758302583025833e-06,
+      "loss": 1.3691,
+      "step": 1655
+    },
+    {
+      "epoch": 2.060983198506534,
+      "grad_norm": 3.1036577224731445,
+      "learning_rate": 1.7735239852398525e-06,
+      "loss": 1.2721,
+      "step": 1656
+    },
+    {
+      "epoch": 2.062227753578096,
+      "grad_norm": 3.604623317718506,
+      "learning_rate": 1.7712177121771217e-06,
+      "loss": 1.376,
+      "step": 1657
+    },
+    {
+      "epoch": 2.063472308649658,
+      "grad_norm": 3.5246458053588867,
+      "learning_rate": 1.7689114391143914e-06,
+      "loss": 1.3891,
+      "step": 1658
+    },
+    {
+      "epoch": 2.0647168637212197,
+      "grad_norm": 3.286940336227417,
+      "learning_rate": 1.7666051660516606e-06,
+      "loss": 1.3978,
+      "step": 1659
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "grad_norm": 3.1022047996520996,
+      "learning_rate": 1.7642988929889302e-06,
+      "loss": 1.3373,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "eval_loss": 1.6144907474517822,
+      "eval_runtime": 48.4382,
+      "eval_samples_per_second": 20.645,
+      "eval_steps_per_second": 0.867,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0672059738643433,
+      "grad_norm": 3.4111568927764893,
+      "learning_rate": 1.7619926199261994e-06,
+      "loss": 1.3816,
+      "step": 1661
+    },
+    {
+      "epoch": 2.068450528935905,
+      "grad_norm": 3.478947162628174,
+      "learning_rate": 1.7596863468634687e-06,
+      "loss": 1.3601,
+      "step": 1662
+    },
+    {
+      "epoch": 2.0696950840074675,
+      "grad_norm": 3.3320810794830322,
+      "learning_rate": 1.7573800738007383e-06,
+      "loss": 1.3454,
+      "step": 1663
+    },
+    {
+      "epoch": 2.0709396390790293,
+      "grad_norm": 3.475529909133911,
+      "learning_rate": 1.7550738007380075e-06,
+      "loss": 1.3666,
+      "step": 1664
+    },
+    {
+      "epoch": 2.072184194150591,
+      "grad_norm": 3.302342653274536,
+      "learning_rate": 1.752767527675277e-06,
+      "loss": 1.3824,
+      "step": 1665
+    },
+    {
+      "epoch": 2.073428749222153,
+      "grad_norm": 3.198641777038574,
+      "learning_rate": 1.7504612546125463e-06,
+      "loss": 1.3398,
+      "step": 1666
+    },
+    {
+      "epoch": 2.074673304293715,
+      "grad_norm": 3.421834945678711,
+      "learning_rate": 1.7481549815498156e-06,
+      "loss": 1.363,
+      "step": 1667
+    },
+    {
+      "epoch": 2.0759178593652767,
+      "grad_norm": 3.654343843460083,
+      "learning_rate": 1.7458487084870852e-06,
+      "loss": 1.3852,
+      "step": 1668
+    },
+    {
+      "epoch": 2.077162414436839,
+      "grad_norm": 3.3543035984039307,
+      "learning_rate": 1.7435424354243544e-06,
+      "loss": 1.3554,
+      "step": 1669
+    },
+    {
+      "epoch": 2.078406969508401,
+      "grad_norm": 3.2144811153411865,
+      "learning_rate": 1.7412361623616236e-06,
+      "loss": 1.3137,
+      "step": 1670
+    },
+    {
+      "epoch": 2.078406969508401,
+      "eval_loss": 1.6155465841293335,
+      "eval_runtime": 52.4111,
+      "eval_samples_per_second": 19.08,
+      "eval_steps_per_second": 0.801,
+      "step": 1670
+    },
+    {
+      "epoch": 2.0796515245799627,
+      "grad_norm": 3.2240991592407227,
+      "learning_rate": 1.7389298892988932e-06,
+      "loss": 1.3578,
+      "step": 1671
+    },
+    {
+      "epoch": 2.0808960796515246,
+      "grad_norm": 3.1450018882751465,
+      "learning_rate": 1.7366236162361625e-06,
+      "loss": 1.2999,
+      "step": 1672
+    },
+    {
+      "epoch": 2.0821406347230864,
+      "grad_norm": 3.4396843910217285,
+      "learning_rate": 1.7343173431734319e-06,
+      "loss": 1.3522,
+      "step": 1673
+    },
+    {
+      "epoch": 2.0833851897946483,
+      "grad_norm": 3.1719677448272705,
+      "learning_rate": 1.7320110701107013e-06,
+      "loss": 1.3552,
+      "step": 1674
+    },
+    {
+      "epoch": 2.08462974486621,
+      "grad_norm": 3.4420199394226074,
+      "learning_rate": 1.7297047970479705e-06,
+      "loss": 1.4033,
+      "step": 1675
+    },
+    {
+      "epoch": 2.0858742999377724,
+      "grad_norm": 3.3366570472717285,
+      "learning_rate": 1.7273985239852401e-06,
+      "loss": 1.3407,
+      "step": 1676
+    },
+    {
+      "epoch": 2.0871188550093343,
+      "grad_norm": 3.3416335582733154,
+      "learning_rate": 1.7250922509225094e-06,
+      "loss": 1.3882,
+      "step": 1677
+    },
+    {
+      "epoch": 2.088363410080896,
+      "grad_norm": 3.4189534187316895,
+      "learning_rate": 1.7227859778597788e-06,
+      "loss": 1.3996,
+      "step": 1678
+    },
+    {
+      "epoch": 2.089607965152458,
+      "grad_norm": 3.309095859527588,
+      "learning_rate": 1.7204797047970482e-06,
+      "loss": 1.3799,
+      "step": 1679
+    },
+    {
+      "epoch": 2.09085252022402,
+      "grad_norm": 3.369511365890503,
+      "learning_rate": 1.7181734317343174e-06,
+      "loss": 1.4159,
+      "step": 1680
+    },
+    {
+      "epoch": 2.09085252022402,
+      "eval_loss": 1.6145448684692383,
+      "eval_runtime": 49.0228,
+      "eval_samples_per_second": 20.399,
+      "eval_steps_per_second": 0.857,
+      "step": 1680
+    },
+    {
+      "epoch": 2.0920970752955816,
+      "grad_norm": 3.323286771774292,
+      "learning_rate": 1.7158671586715868e-06,
+      "loss": 1.4035,
+      "step": 1681
+    },
+    {
+      "epoch": 2.093341630367144,
+      "grad_norm": 3.3053157329559326,
+      "learning_rate": 1.7135608856088563e-06,
+      "loss": 1.3498,
+      "step": 1682
+    },
+    {
+      "epoch": 2.094586185438706,
+      "grad_norm": 3.141890048980713,
+      "learning_rate": 1.7112546125461257e-06,
+      "loss": 1.3438,
+      "step": 1683
+    },
+    {
+      "epoch": 2.0958307405102676,
+      "grad_norm": 3.135058879852295,
+      "learning_rate": 1.708948339483395e-06,
+      "loss": 1.3726,
+      "step": 1684
+    },
+    {
+      "epoch": 2.0970752955818295,
+      "grad_norm": 3.2389063835144043,
+      "learning_rate": 1.7066420664206643e-06,
+      "loss": 1.3661,
+      "step": 1685
+    },
+    {
+      "epoch": 2.0983198506533913,
+      "grad_norm": 3.1741721630096436,
+      "learning_rate": 1.7043357933579337e-06,
+      "loss": 1.3438,
+      "step": 1686
+    },
+    {
+      "epoch": 2.099564405724953,
+      "grad_norm": 3.3524587154388428,
+      "learning_rate": 1.7020295202952032e-06,
+      "loss": 1.3388,
+      "step": 1687
+    },
+    {
+      "epoch": 2.100808960796515,
+      "grad_norm": 3.3574740886688232,
+      "learning_rate": 1.6997232472324724e-06,
+      "loss": 1.3702,
+      "step": 1688
+    },
+    {
+      "epoch": 2.1020535158680773,
+      "grad_norm": 3.458645820617676,
+      "learning_rate": 1.6974169741697418e-06,
+      "loss": 1.3785,
+      "step": 1689
+    },
+    {
+      "epoch": 2.103298070939639,
+      "grad_norm": 3.346677303314209,
+      "learning_rate": 1.6951107011070112e-06,
+      "loss": 1.3295,
+      "step": 1690
+    },
+    {
+      "epoch": 2.103298070939639,
+      "eval_loss": 1.6184555292129517,
+      "eval_runtime": 46.6544,
+      "eval_samples_per_second": 21.434,
+      "eval_steps_per_second": 0.9,
+      "step": 1690
+    },
+    {
+      "epoch": 2.104542626011201,
+      "grad_norm": 3.3403804302215576,
+      "learning_rate": 1.6928044280442806e-06,
+      "loss": 1.3912,
+      "step": 1691
+    },
+    {
+      "epoch": 2.105787181082763,
+      "grad_norm": 3.2638344764709473,
+      "learning_rate": 1.6904981549815499e-06,
+      "loss": 1.3684,
+      "step": 1692
+    },
+    {
+      "epoch": 2.1070317361543247,
+      "grad_norm": 3.192701578140259,
+      "learning_rate": 1.6881918819188193e-06,
+      "loss": 1.2997,
+      "step": 1693
+    },
+    {
+      "epoch": 2.1082762912258866,
+      "grad_norm": 3.482121706008911,
+      "learning_rate": 1.6858856088560887e-06,
+      "loss": 1.3515,
+      "step": 1694
+    },
+    {
+      "epoch": 2.109520846297449,
+      "grad_norm": 3.292353630065918,
+      "learning_rate": 1.6835793357933581e-06,
+      "loss": 1.3603,
+      "step": 1695
+    },
+    {
+      "epoch": 2.1107654013690107,
+      "grad_norm": 3.34220290184021,
+      "learning_rate": 1.6812730627306275e-06,
+      "loss": 1.3666,
+      "step": 1696
+    },
+    {
+      "epoch": 2.1120099564405725,
+      "grad_norm": 3.190058469772339,
+      "learning_rate": 1.6789667896678968e-06,
+      "loss": 1.3549,
+      "step": 1697
+    },
+    {
+      "epoch": 2.1132545115121344,
+      "grad_norm": 3.3759775161743164,
+      "learning_rate": 1.6766605166051662e-06,
+      "loss": 1.3628,
+      "step": 1698
+    },
+    {
+      "epoch": 2.1144990665836962,
+      "grad_norm": 3.419440507888794,
+      "learning_rate": 1.6743542435424356e-06,
+      "loss": 1.3618,
+      "step": 1699
+    },
+    {
+      "epoch": 2.115743621655258,
+      "grad_norm": 3.5021324157714844,
+      "learning_rate": 1.6720479704797048e-06,
+      "loss": 1.4023,
+      "step": 1700
+    },
+    {
+      "epoch": 2.115743621655258,
+      "eval_loss": 1.6218053102493286,
+      "eval_runtime": 48.3483,
+      "eval_samples_per_second": 20.683,
+      "eval_steps_per_second": 0.869,
+      "step": 1700
+    },
+    {
+      "epoch": 2.11698817672682,
+      "grad_norm": 3.449636936187744,
+      "learning_rate": 1.6697416974169742e-06,
+      "loss": 1.3534,
+      "step": 1701
+    },
+    {
+      "epoch": 2.1182327317983822,
+      "grad_norm": 3.483173370361328,
+      "learning_rate": 1.6674354243542437e-06,
+      "loss": 1.4053,
+      "step": 1702
+    },
+    {
+      "epoch": 2.119477286869944,
+      "grad_norm": 3.3713502883911133,
+      "learning_rate": 1.665129151291513e-06,
+      "loss": 1.3352,
+      "step": 1703
+    },
+    {
+      "epoch": 2.120721841941506,
+      "grad_norm": 3.2565910816192627,
+      "learning_rate": 1.6628228782287825e-06,
+      "loss": 1.3751,
+      "step": 1704
+    },
+    {
+      "epoch": 2.1219663970130678,
+      "grad_norm": 3.1434268951416016,
+      "learning_rate": 1.6605166051660517e-06,
+      "loss": 1.346,
+      "step": 1705
+    },
+    {
+      "epoch": 2.1232109520846296,
+      "grad_norm": 3.229909896850586,
+      "learning_rate": 1.6582103321033211e-06,
+      "loss": 1.3477,
+      "step": 1706
+    },
+    {
+      "epoch": 2.1244555071561915,
+      "grad_norm": 3.3930044174194336,
+      "learning_rate": 1.6559040590405906e-06,
+      "loss": 1.3374,
+      "step": 1707
+    },
+    {
+      "epoch": 2.1257000622277538,
+      "grad_norm": 3.2674202919006348,
+      "learning_rate": 1.6535977859778598e-06,
+      "loss": 1.321,
+      "step": 1708
+    },
+    {
+      "epoch": 2.1269446172993156,
+      "grad_norm": 3.3048641681671143,
+      "learning_rate": 1.6512915129151294e-06,
+      "loss": 1.3641,
+      "step": 1709
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "grad_norm": 3.4714198112487793,
+      "learning_rate": 1.6489852398523986e-06,
+      "loss": 1.3931,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "eval_loss": 1.6198205947875977,
+      "eval_runtime": 42.9272,
+      "eval_samples_per_second": 23.295,
+      "eval_steps_per_second": 0.978,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1294337274424393,
+      "grad_norm": 3.440058946609497,
+      "learning_rate": 1.6466789667896678e-06,
+      "loss": 1.3892,
+      "step": 1711
+    },
+    {
+      "epoch": 2.130678282514001,
+      "grad_norm": 3.4067635536193848,
+      "learning_rate": 1.6443726937269375e-06,
+      "loss": 1.3719,
+      "step": 1712
+    },
+    {
+      "epoch": 2.131922837585563,
+      "grad_norm": 3.285536766052246,
+      "learning_rate": 1.6420664206642067e-06,
+      "loss": 1.3735,
+      "step": 1713
+    },
+    {
+      "epoch": 2.133167392657125,
+      "grad_norm": 3.171344041824341,
+      "learning_rate": 1.6397601476014763e-06,
+      "loss": 1.3628,
+      "step": 1714
+    },
+    {
+      "epoch": 2.134411947728687,
+      "grad_norm": 3.3051321506500244,
+      "learning_rate": 1.6374538745387455e-06,
+      "loss": 1.3579,
+      "step": 1715
+    },
+    {
+      "epoch": 2.135656502800249,
+      "grad_norm": 3.2083475589752197,
+      "learning_rate": 1.6351476014760147e-06,
+      "loss": 1.297,
+      "step": 1716
+    },
+    {
+      "epoch": 2.136901057871811,
+      "grad_norm": 3.340172052383423,
+      "learning_rate": 1.6328413284132844e-06,
+      "loss": 1.3468,
+      "step": 1717
+    },
+    {
+      "epoch": 2.1381456129433727,
+      "grad_norm": 3.3475770950317383,
+      "learning_rate": 1.6305350553505536e-06,
+      "loss": 1.3918,
+      "step": 1718
+    },
+    {
+      "epoch": 2.1393901680149345,
+      "grad_norm": 3.4087703227996826,
+      "learning_rate": 1.6282287822878228e-06,
+      "loss": 1.4083,
+      "step": 1719
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "grad_norm": 3.461160898208618,
+      "learning_rate": 1.6259225092250924e-06,
+      "loss": 1.3471,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "eval_loss": 1.6216135025024414,
+      "eval_runtime": 49.1097,
+      "eval_samples_per_second": 20.363,
+      "eval_steps_per_second": 0.855,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1418792781580587,
+      "grad_norm": 3.3619799613952637,
+      "learning_rate": 1.6236162361623616e-06,
+      "loss": 1.3235,
+      "step": 1721
+    },
+    {
+      "epoch": 2.1431238332296205,
+      "grad_norm": 3.332010507583618,
+      "learning_rate": 1.6213099630996313e-06,
+      "loss": 1.3576,
+      "step": 1722
+    },
+    {
+      "epoch": 2.1443683883011824,
+      "grad_norm": 3.300055503845215,
+      "learning_rate": 1.6190036900369005e-06,
+      "loss": 1.3269,
+      "step": 1723
+    },
+    {
+      "epoch": 2.145612943372744,
+      "grad_norm": 3.3326926231384277,
+      "learning_rate": 1.6166974169741697e-06,
+      "loss": 1.346,
+      "step": 1724
+    },
+    {
+      "epoch": 2.146857498444306,
+      "grad_norm": 3.4713706970214844,
+      "learning_rate": 1.6143911439114393e-06,
+      "loss": 1.3416,
+      "step": 1725
+    },
+    {
+      "epoch": 2.148102053515868,
+      "grad_norm": 3.1899967193603516,
+      "learning_rate": 1.6120848708487085e-06,
+      "loss": 1.3192,
+      "step": 1726
+    },
+    {
+      "epoch": 2.1493466085874298,
+      "grad_norm": 3.1781857013702393,
+      "learning_rate": 1.6097785977859782e-06,
+      "loss": 1.3197,
+      "step": 1727
+    },
+    {
+      "epoch": 2.150591163658992,
+      "grad_norm": 3.3369524478912354,
+      "learning_rate": 1.6074723247232474e-06,
+      "loss": 1.3804,
+      "step": 1728
+    },
+    {
+      "epoch": 2.151835718730554,
+      "grad_norm": 3.2813405990600586,
+      "learning_rate": 1.6051660516605166e-06,
+      "loss": 1.3778,
+      "step": 1729
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "grad_norm": 3.3395135402679443,
+      "learning_rate": 1.6028597785977862e-06,
+      "loss": 1.3523,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "eval_loss": 1.6181366443634033,
+      "eval_runtime": 50.7133,
+      "eval_samples_per_second": 19.719,
+      "eval_steps_per_second": 0.828,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1543248288736776,
+      "grad_norm": 3.2905092239379883,
+      "learning_rate": 1.6005535055350554e-06,
+      "loss": 1.3568,
+      "step": 1731
+    },
+    {
+      "epoch": 2.1555693839452394,
+      "grad_norm": 3.314086437225342,
+      "learning_rate": 1.5982472324723247e-06,
+      "loss": 1.3543,
+      "step": 1732
+    },
+    {
+      "epoch": 2.1568139390168013,
+      "grad_norm": 3.340034008026123,
+      "learning_rate": 1.5959409594095943e-06,
+      "loss": 1.3639,
+      "step": 1733
+    },
+    {
+      "epoch": 2.1580584940883636,
+      "grad_norm": 3.264814615249634,
+      "learning_rate": 1.5936346863468635e-06,
+      "loss": 1.3599,
+      "step": 1734
+    },
+    {
+      "epoch": 2.1593030491599254,
+      "grad_norm": 3.2429778575897217,
+      "learning_rate": 1.5913284132841331e-06,
+      "loss": 1.3379,
+      "step": 1735
+    },
+    {
+      "epoch": 2.1605476042314873,
+      "grad_norm": 3.221935987472534,
+      "learning_rate": 1.5890221402214023e-06,
+      "loss": 1.3162,
+      "step": 1736
+    },
+    {
+      "epoch": 2.161792159303049,
+      "grad_norm": 3.266073703765869,
+      "learning_rate": 1.5867158671586716e-06,
+      "loss": 1.3632,
+      "step": 1737
+    },
+    {
+      "epoch": 2.163036714374611,
+      "grad_norm": 3.2618112564086914,
+      "learning_rate": 1.5844095940959412e-06,
+      "loss": 1.3779,
+      "step": 1738
+    },
+    {
+      "epoch": 2.164281269446173,
+      "grad_norm": 3.2474825382232666,
+      "learning_rate": 1.5821033210332104e-06,
+      "loss": 1.3201,
+      "step": 1739
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "grad_norm": 3.4494247436523438,
+      "learning_rate": 1.57979704797048e-06,
+      "loss": 1.3949,
+      "step": 1740
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "eval_loss": 1.6180604696273804,
+      "eval_runtime": 49.3152,
+      "eval_samples_per_second": 20.278,
+      "eval_steps_per_second": 0.852,
+      "step": 1740
+    },
+    {
+      "epoch": 2.166770379589297,
+      "grad_norm": 3.221304416656494,
+      "learning_rate": 1.5774907749077492e-06,
+      "loss": 1.3341,
+      "step": 1741
+    },
+    {
+      "epoch": 2.168014934660859,
+      "grad_norm": 3.28930401802063,
+      "learning_rate": 1.5751845018450185e-06,
+      "loss": 1.3151,
+      "step": 1742
+    },
+    {
+      "epoch": 2.1692594897324207,
+      "grad_norm": 3.3582026958465576,
+      "learning_rate": 1.572878228782288e-06,
+      "loss": 1.3767,
+      "step": 1743
+    },
+    {
+      "epoch": 2.1705040448039825,
+      "grad_norm": 3.2603683471679688,
+      "learning_rate": 1.5705719557195573e-06,
+      "loss": 1.3644,
+      "step": 1744
+    },
+    {
+      "epoch": 2.1717485998755444,
+      "grad_norm": 3.3291070461273193,
+      "learning_rate": 1.5682656826568267e-06,
+      "loss": 1.3226,
+      "step": 1745
+    },
+    {
+      "epoch": 2.172993154947106,
+      "grad_norm": 3.473127841949463,
+      "learning_rate": 1.5659594095940962e-06,
+      "loss": 1.3531,
+      "step": 1746
+    },
+    {
+      "epoch": 2.1742377100186685,
+      "grad_norm": 3.2014408111572266,
+      "learning_rate": 1.5636531365313654e-06,
+      "loss": 1.3634,
+      "step": 1747
+    },
+    {
+      "epoch": 2.1754822650902303,
+      "grad_norm": 3.56447434425354,
+      "learning_rate": 1.561346863468635e-06,
+      "loss": 1.3788,
+      "step": 1748
+    },
+    {
+      "epoch": 2.176726820161792,
+      "grad_norm": 3.351318359375,
+      "learning_rate": 1.5590405904059042e-06,
+      "loss": 1.3133,
+      "step": 1749
+    },
+    {
+      "epoch": 2.177971375233354,
+      "grad_norm": 3.278183937072754,
+      "learning_rate": 1.5567343173431734e-06,
+      "loss": 1.3164,
+      "step": 1750
+    },
+    {
+      "epoch": 2.177971375233354,
+      "eval_loss": 1.6197904348373413,
+      "eval_runtime": 50.2776,
+      "eval_samples_per_second": 19.89,
+      "eval_steps_per_second": 0.835,
+      "step": 1750
+    },
+    {
+      "epoch": 2.179215930304916,
+      "grad_norm": 3.379854917526245,
+      "learning_rate": 1.554428044280443e-06,
+      "loss": 1.3785,
+      "step": 1751
+    },
+    {
+      "epoch": 2.1804604853764777,
+      "grad_norm": 3.2265610694885254,
+      "learning_rate": 1.5521217712177123e-06,
+      "loss": 1.3483,
+      "step": 1752
+    },
+    {
+      "epoch": 2.18170504044804,
+      "grad_norm": 3.2259609699249268,
+      "learning_rate": 1.5498154981549817e-06,
+      "loss": 1.377,
+      "step": 1753
+    },
+    {
+      "epoch": 2.182949595519602,
+      "grad_norm": 3.1671273708343506,
+      "learning_rate": 1.5475092250922511e-06,
+      "loss": 1.3106,
+      "step": 1754
+    },
+    {
+      "epoch": 2.1841941505911637,
+      "grad_norm": 3.280299425125122,
+      "learning_rate": 1.5452029520295203e-06,
+      "loss": 1.3271,
+      "step": 1755
+    },
+    {
+      "epoch": 2.1854387056627256,
+      "grad_norm": 3.224238634109497,
+      "learning_rate": 1.5428966789667897e-06,
+      "loss": 1.3369,
+      "step": 1756
+    },
+    {
+      "epoch": 2.1866832607342874,
+      "grad_norm": 3.3139021396636963,
+      "learning_rate": 1.5405904059040592e-06,
+      "loss": 1.3562,
+      "step": 1757
+    },
+    {
+      "epoch": 2.1879278158058493,
+      "grad_norm": 3.340569257736206,
+      "learning_rate": 1.5382841328413286e-06,
+      "loss": 1.3526,
+      "step": 1758
+    },
+    {
+      "epoch": 2.189172370877411,
+      "grad_norm": 3.505159854888916,
+      "learning_rate": 1.535977859778598e-06,
+      "loss": 1.3796,
+      "step": 1759
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "grad_norm": 3.351260185241699,
+      "learning_rate": 1.5336715867158672e-06,
+      "loss": 1.3676,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "eval_loss": 1.6169729232788086,
+      "eval_runtime": 48.0801,
+      "eval_samples_per_second": 20.799,
+      "eval_steps_per_second": 0.874,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1916614810205353,
+      "grad_norm": 3.4291462898254395,
+      "learning_rate": 1.5313653136531366e-06,
+      "loss": 1.3574,
+      "step": 1761
+    },
+    {
+      "epoch": 2.192906036092097,
+      "grad_norm": 3.5459227561950684,
+      "learning_rate": 1.529059040590406e-06,
+      "loss": 1.3919,
+      "step": 1762
+    },
+    {
+      "epoch": 2.194150591163659,
+      "grad_norm": 3.298430919647217,
+      "learning_rate": 1.5267527675276755e-06,
+      "loss": 1.3551,
+      "step": 1763
+    },
+    {
+      "epoch": 2.195395146235221,
+      "grad_norm": 3.4521286487579346,
+      "learning_rate": 1.5244464944649447e-06,
+      "loss": 1.3779,
+      "step": 1764
+    },
+    {
+      "epoch": 2.1966397013067827,
+      "grad_norm": 3.354443073272705,
+      "learning_rate": 1.5221402214022141e-06,
+      "loss": 1.3574,
+      "step": 1765
+    },
+    {
+      "epoch": 2.197884256378345,
+      "grad_norm": 3.069854259490967,
+      "learning_rate": 1.5198339483394836e-06,
+      "loss": 1.2987,
+      "step": 1766
+    },
+    {
+      "epoch": 2.199128811449907,
+      "grad_norm": 3.0894274711608887,
+      "learning_rate": 1.517527675276753e-06,
+      "loss": 1.3408,
+      "step": 1767
+    },
+    {
+      "epoch": 2.2003733665214686,
+      "grad_norm": 3.3033676147460938,
+      "learning_rate": 1.5152214022140222e-06,
+      "loss": 1.3394,
+      "step": 1768
+    },
+    {
+      "epoch": 2.2016179215930305,
+      "grad_norm": 3.2894108295440674,
+      "learning_rate": 1.5129151291512916e-06,
+      "loss": 1.3505,
+      "step": 1769
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "grad_norm": 3.263564109802246,
+      "learning_rate": 1.510608856088561e-06,
+      "loss": 1.3318,
+      "step": 1770
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "eval_loss": 1.6183674335479736,
+      "eval_runtime": 48.3189,
+      "eval_samples_per_second": 20.696,
+      "eval_steps_per_second": 0.869,
+      "step": 1770
+    },
+    {
+      "epoch": 2.204107031736154,
+      "grad_norm": 3.2933449745178223,
+      "learning_rate": 1.5083025830258305e-06,
+      "loss": 1.3397,
+      "step": 1771
+    },
+    {
+      "epoch": 2.205351586807716,
+      "grad_norm": 3.3185815811157227,
+      "learning_rate": 1.5059963099630997e-06,
+      "loss": 1.3339,
+      "step": 1772
+    },
+    {
+      "epoch": 2.2065961418792783,
+      "grad_norm": 3.445251703262329,
+      "learning_rate": 1.503690036900369e-06,
+      "loss": 1.3898,
+      "step": 1773
+    },
+    {
+      "epoch": 2.20784069695084,
+      "grad_norm": 3.399627685546875,
+      "learning_rate": 1.5013837638376385e-06,
+      "loss": 1.3646,
+      "step": 1774
+    },
+    {
+      "epoch": 2.209085252022402,
+      "grad_norm": 3.3483574390411377,
+      "learning_rate": 1.499077490774908e-06,
+      "loss": 1.3732,
+      "step": 1775
+    },
+    {
+      "epoch": 2.210329807093964,
+      "grad_norm": 3.3097689151763916,
+      "learning_rate": 1.4967712177121774e-06,
+      "loss": 1.3317,
+      "step": 1776
+    },
+    {
+      "epoch": 2.2115743621655257,
+      "grad_norm": 3.254025459289551,
+      "learning_rate": 1.4944649446494466e-06,
+      "loss": 1.3818,
+      "step": 1777
+    },
+    {
+      "epoch": 2.2128189172370876,
+      "grad_norm": 3.2063064575195312,
+      "learning_rate": 1.492158671586716e-06,
+      "loss": 1.3357,
+      "step": 1778
+    },
+    {
+      "epoch": 2.21406347230865,
+      "grad_norm": 3.1974005699157715,
+      "learning_rate": 1.4898523985239854e-06,
+      "loss": 1.4103,
+      "step": 1779
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "grad_norm": 3.1551153659820557,
+      "learning_rate": 1.4875461254612546e-06,
+      "loss": 1.3088,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "eval_loss": 1.6195781230926514,
+      "eval_runtime": 52.379,
+      "eval_samples_per_second": 19.092,
+      "eval_steps_per_second": 0.802,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2165525824517736,
+      "grad_norm": 3.3186404705047607,
+      "learning_rate": 1.485239852398524e-06,
+      "loss": 1.3775,
+      "step": 1781
+    },
+    {
+      "epoch": 2.2177971375233354,
+      "grad_norm": 3.2764358520507812,
+      "learning_rate": 1.4829335793357935e-06,
+      "loss": 1.3253,
+      "step": 1782
+    },
+    {
+      "epoch": 2.2190416925948973,
+      "grad_norm": 3.3535802364349365,
+      "learning_rate": 1.4806273062730627e-06,
+      "loss": 1.3451,
+      "step": 1783
+    },
+    {
+      "epoch": 2.220286247666459,
+      "grad_norm": 3.304255962371826,
+      "learning_rate": 1.4783210332103323e-06,
+      "loss": 1.3565,
+      "step": 1784
+    },
+    {
+      "epoch": 2.2215308027380214,
+      "grad_norm": 3.281244993209839,
+      "learning_rate": 1.4760147601476015e-06,
+      "loss": 1.3458,
+      "step": 1785
+    },
+    {
+      "epoch": 2.2227753578095832,
+      "grad_norm": 3.4199814796447754,
+      "learning_rate": 1.473708487084871e-06,
+      "loss": 1.359,
+      "step": 1786
+    },
+    {
+      "epoch": 2.224019912881145,
+      "grad_norm": 3.298577070236206,
+      "learning_rate": 1.4714022140221404e-06,
+      "loss": 1.3298,
+      "step": 1787
+    },
+    {
+      "epoch": 2.225264467952707,
+      "grad_norm": 3.2206099033355713,
+      "learning_rate": 1.4690959409594096e-06,
+      "loss": 1.3026,
+      "step": 1788
+    },
+    {
+      "epoch": 2.226509023024269,
+      "grad_norm": 3.3583920001983643,
+      "learning_rate": 1.4667896678966792e-06,
+      "loss": 1.3618,
+      "step": 1789
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "grad_norm": 3.4446728229522705,
+      "learning_rate": 1.4644833948339484e-06,
+      "loss": 1.3895,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "eval_loss": 1.6163197755813599,
+      "eval_runtime": 49.4171,
+      "eval_samples_per_second": 20.236,
+      "eval_steps_per_second": 0.85,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2289981331673925,
+      "grad_norm": 3.329155445098877,
+      "learning_rate": 1.4621771217712176e-06,
+      "loss": 1.3368,
+      "step": 1791
+    },
+    {
+      "epoch": 2.2302426882389548,
+      "grad_norm": 3.4599087238311768,
+      "learning_rate": 1.4598708487084873e-06,
+      "loss": 1.3487,
+      "step": 1792
+    },
+    {
+      "epoch": 2.2314872433105166,
+      "grad_norm": 3.3162965774536133,
+      "learning_rate": 1.4575645756457565e-06,
+      "loss": 1.3493,
+      "step": 1793
+    },
+    {
+      "epoch": 2.2327317983820785,
+      "grad_norm": 3.1206021308898926,
+      "learning_rate": 1.4552583025830261e-06,
+      "loss": 1.3506,
+      "step": 1794
+    },
+    {
+      "epoch": 2.2339763534536403,
+      "grad_norm": 3.334165573120117,
+      "learning_rate": 1.4529520295202953e-06,
+      "loss": 1.3315,
+      "step": 1795
+    },
+    {
+      "epoch": 2.235220908525202,
+      "grad_norm": 3.3404204845428467,
+      "learning_rate": 1.4506457564575645e-06,
+      "loss": 1.3163,
+      "step": 1796
+    },
+    {
+      "epoch": 2.236465463596764,
+      "grad_norm": 3.2753002643585205,
+      "learning_rate": 1.4483394833948342e-06,
+      "loss": 1.3567,
+      "step": 1797
+    },
+    {
+      "epoch": 2.2377100186683263,
+      "grad_norm": 3.42242431640625,
+      "learning_rate": 1.4460332103321034e-06,
+      "loss": 1.3761,
+      "step": 1798
+    },
+    {
+      "epoch": 2.238954573739888,
+      "grad_norm": 3.31380033493042,
+      "learning_rate": 1.4437269372693726e-06,
+      "loss": 1.3353,
+      "step": 1799
+    },
+    {
+      "epoch": 2.24019912881145,
+      "grad_norm": 3.239811658859253,
+      "learning_rate": 1.4414206642066422e-06,
+      "loss": 1.3187,
+      "step": 1800
+    },
+    {
+      "epoch": 2.24019912881145,
+      "eval_loss": 1.6201411485671997,
+      "eval_runtime": 53.5369,
+      "eval_samples_per_second": 18.679,
+      "eval_steps_per_second": 0.785,
+      "step": 1800
+    },
+    {
+      "epoch": 2.241443683883012,
+      "grad_norm": 3.549010753631592,
+      "learning_rate": 1.4391143911439114e-06,
+      "loss": 1.3633,
+      "step": 1801
+    },
+    {
+      "epoch": 2.2426882389545737,
+      "grad_norm": 3.1888253688812256,
+      "learning_rate": 1.436808118081181e-06,
+      "loss": 1.3292,
+      "step": 1802
+    },
+    {
+      "epoch": 2.2439327940261355,
+      "grad_norm": 3.1015641689300537,
+      "learning_rate": 1.4345018450184503e-06,
+      "loss": 1.3404,
+      "step": 1803
+    },
+    {
+      "epoch": 2.2451773490976974,
+      "grad_norm": 3.2635490894317627,
+      "learning_rate": 1.4321955719557195e-06,
+      "loss": 1.3181,
+      "step": 1804
+    },
+    {
+      "epoch": 2.2464219041692597,
+      "grad_norm": 3.314697265625,
+      "learning_rate": 1.4298892988929891e-06,
+      "loss": 1.308,
+      "step": 1805
+    },
+    {
+      "epoch": 2.2476664592408215,
+      "grad_norm": 3.3572986125946045,
+      "learning_rate": 1.4275830258302584e-06,
+      "loss": 1.2888,
+      "step": 1806
+    },
+    {
+      "epoch": 2.2489110143123834,
+      "grad_norm": 3.403423547744751,
+      "learning_rate": 1.425276752767528e-06,
+      "loss": 1.3438,
+      "step": 1807
+    },
+    {
+      "epoch": 2.2501555693839452,
+      "grad_norm": 3.5180821418762207,
+      "learning_rate": 1.4229704797047972e-06,
+      "loss": 1.3613,
+      "step": 1808
+    },
+    {
+      "epoch": 2.251400124455507,
+      "grad_norm": 3.5315089225769043,
+      "learning_rate": 1.4206642066420664e-06,
+      "loss": 1.3655,
+      "step": 1809
+    },
+    {
+      "epoch": 2.252644679527069,
+      "grad_norm": 3.4266903400421143,
+      "learning_rate": 1.418357933579336e-06,
+      "loss": 1.3481,
+      "step": 1810
+    },
+    {
+      "epoch": 2.252644679527069,
+      "eval_loss": 1.619051456451416,
+      "eval_runtime": 43.1595,
+      "eval_samples_per_second": 23.17,
+      "eval_steps_per_second": 0.973,
+      "step": 1810
+    },
+    {
+      "epoch": 2.253889234598631,
+      "grad_norm": 3.412053108215332,
+      "learning_rate": 1.4160516605166053e-06,
+      "loss": 1.3097,
+      "step": 1811
+    },
+    {
+      "epoch": 2.255133789670193,
+      "grad_norm": 3.4371933937072754,
+      "learning_rate": 1.4137453874538745e-06,
+      "loss": 1.3831,
+      "step": 1812
+    },
+    {
+      "epoch": 2.256378344741755,
+      "grad_norm": 3.331648588180542,
+      "learning_rate": 1.411439114391144e-06,
+      "loss": 1.3701,
+      "step": 1813
+    },
+    {
+      "epoch": 2.2576228998133168,
+      "grad_norm": 3.3840672969818115,
+      "learning_rate": 1.4091328413284133e-06,
+      "loss": 1.3676,
+      "step": 1814
+    },
+    {
+      "epoch": 2.2588674548848786,
+      "grad_norm": 3.3289988040924072,
+      "learning_rate": 1.406826568265683e-06,
+      "loss": 1.3281,
+      "step": 1815
+    },
+    {
+      "epoch": 2.2601120099564405,
+      "grad_norm": 3.309511423110962,
+      "learning_rate": 1.4045202952029522e-06,
+      "loss": 1.3216,
+      "step": 1816
+    },
+    {
+      "epoch": 2.2613565650280023,
+      "grad_norm": 3.3434019088745117,
+      "learning_rate": 1.4022140221402214e-06,
+      "loss": 1.3261,
+      "step": 1817
+    },
+    {
+      "epoch": 2.2626011200995646,
+      "grad_norm": 3.3371620178222656,
+      "learning_rate": 1.399907749077491e-06,
+      "loss": 1.3597,
+      "step": 1818
+    },
+    {
+      "epoch": 2.2638456751711264,
+      "grad_norm": 3.287353515625,
+      "learning_rate": 1.3976014760147602e-06,
+      "loss": 1.3431,
+      "step": 1819
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "grad_norm": 3.422102689743042,
+      "learning_rate": 1.3952952029520298e-06,
+      "loss": 1.3454,
+      "step": 1820
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "eval_loss": 1.6200023889541626,
+      "eval_runtime": 46.3983,
+      "eval_samples_per_second": 21.552,
+      "eval_steps_per_second": 0.905,
+      "step": 1820
+    },
+    {
+      "epoch": 2.26633478531425,
+      "grad_norm": 3.6252498626708984,
+      "learning_rate": 1.392988929889299e-06,
+      "loss": 1.4066,
+      "step": 1821
+    },
+    {
+      "epoch": 2.267579340385812,
+      "grad_norm": 3.423708915710449,
+      "learning_rate": 1.3906826568265683e-06,
+      "loss": 1.3504,
+      "step": 1822
+    },
+    {
+      "epoch": 2.268823895457374,
+      "grad_norm": 3.3063361644744873,
+      "learning_rate": 1.388376383763838e-06,
+      "loss": 1.3635,
+      "step": 1823
+    },
+    {
+      "epoch": 2.270068450528936,
+      "grad_norm": 3.380657911300659,
+      "learning_rate": 1.3860701107011071e-06,
+      "loss": 1.3442,
+      "step": 1824
+    },
+    {
+      "epoch": 2.271313005600498,
+      "grad_norm": 3.300102472305298,
+      "learning_rate": 1.3837638376383765e-06,
+      "loss": 1.3741,
+      "step": 1825
+    },
+    {
+      "epoch": 2.27255756067206,
+      "grad_norm": 3.3895652294158936,
+      "learning_rate": 1.381457564575646e-06,
+      "loss": 1.3288,
+      "step": 1826
+    },
+    {
+      "epoch": 2.2738021157436217,
+      "grad_norm": 3.4393489360809326,
+      "learning_rate": 1.3791512915129152e-06,
+      "loss": 1.3544,
+      "step": 1827
+    },
+    {
+      "epoch": 2.2750466708151835,
+      "grad_norm": 3.4510536193847656,
+      "learning_rate": 1.3768450184501846e-06,
+      "loss": 1.3894,
+      "step": 1828
+    },
+    {
+      "epoch": 2.2762912258867454,
+      "grad_norm": 3.39282488822937,
+      "learning_rate": 1.374538745387454e-06,
+      "loss": 1.3776,
+      "step": 1829
+    },
+    {
+      "epoch": 2.277535780958307,
+      "grad_norm": 3.250432014465332,
+      "learning_rate": 1.3722324723247232e-06,
+      "loss": 1.3511,
+      "step": 1830
+    },
+    {
+      "epoch": 2.277535780958307,
+      "eval_loss": 1.6178572177886963,
+      "eval_runtime": 45.3086,
+      "eval_samples_per_second": 22.071,
+      "eval_steps_per_second": 0.927,
+      "step": 1830
+    },
+    {
+      "epoch": 2.2787803360298695,
+      "grad_norm": 3.4044647216796875,
+      "learning_rate": 1.3699261992619929e-06,
+      "loss": 1.3548,
+      "step": 1831
+    },
+    {
+      "epoch": 2.2800248911014314,
+      "grad_norm": 3.4996840953826904,
+      "learning_rate": 1.367619926199262e-06,
+      "loss": 1.4069,
+      "step": 1832
+    },
+    {
+      "epoch": 2.281269446172993,
+      "grad_norm": 3.4036638736724854,
+      "learning_rate": 1.3653136531365315e-06,
+      "loss": 1.3499,
+      "step": 1833
+    },
+    {
+      "epoch": 2.282514001244555,
+      "grad_norm": 3.328805685043335,
+      "learning_rate": 1.363007380073801e-06,
+      "loss": 1.3266,
+      "step": 1834
+    },
+    {
+      "epoch": 2.283758556316117,
+      "grad_norm": 3.410924196243286,
+      "learning_rate": 1.3607011070110701e-06,
+      "loss": 1.3303,
+      "step": 1835
+    },
+    {
+      "epoch": 2.2850031113876788,
+      "grad_norm": 3.323162794113159,
+      "learning_rate": 1.3583948339483396e-06,
+      "loss": 1.3284,
+      "step": 1836
+    },
+    {
+      "epoch": 2.286247666459241,
+      "grad_norm": 3.3356032371520996,
+      "learning_rate": 1.356088560885609e-06,
+      "loss": 1.2842,
+      "step": 1837
+    },
+    {
+      "epoch": 2.287492221530803,
+      "grad_norm": 3.3366599082946777,
+      "learning_rate": 1.3537822878228784e-06,
+      "loss": 1.321,
+      "step": 1838
+    },
+    {
+      "epoch": 2.2887367766023647,
+      "grad_norm": 3.3824799060821533,
+      "learning_rate": 1.3514760147601478e-06,
+      "loss": 1.3744,
+      "step": 1839
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "grad_norm": 3.6268486976623535,
+      "learning_rate": 1.349169741697417e-06,
+      "loss": 1.4097,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "eval_loss": 1.6148018836975098,
+      "eval_runtime": 48.3459,
+      "eval_samples_per_second": 20.684,
+      "eval_steps_per_second": 0.869,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2912258867454884,
+      "grad_norm": 3.3530380725860596,
+      "learning_rate": 1.3468634686346865e-06,
+      "loss": 1.3924,
+      "step": 1841
+    },
+    {
+      "epoch": 2.2924704418170503,
+      "grad_norm": 3.456149101257324,
+      "learning_rate": 1.3445571955719559e-06,
+      "loss": 1.3837,
+      "step": 1842
+    },
+    {
+      "epoch": 2.293714996888612,
+      "grad_norm": 3.37180495262146,
+      "learning_rate": 1.3422509225092253e-06,
+      "loss": 1.3411,
+      "step": 1843
+    },
+    {
+      "epoch": 2.2949595519601744,
+      "grad_norm": 3.286172389984131,
+      "learning_rate": 1.3399446494464945e-06,
+      "loss": 1.2888,
+      "step": 1844
+    },
+    {
+      "epoch": 2.2962041070317363,
+      "grad_norm": 3.3966121673583984,
+      "learning_rate": 1.337638376383764e-06,
+      "loss": 1.3644,
+      "step": 1845
+    },
+    {
+      "epoch": 2.297448662103298,
+      "grad_norm": 3.4066531658172607,
+      "learning_rate": 1.3353321033210334e-06,
+      "loss": 1.3543,
+      "step": 1846
+    },
+    {
+      "epoch": 2.29869321717486,
+      "grad_norm": 3.542680501937866,
+      "learning_rate": 1.3330258302583028e-06,
+      "loss": 1.3823,
+      "step": 1847
+    },
+    {
+      "epoch": 2.299937772246422,
+      "grad_norm": 3.323636770248413,
+      "learning_rate": 1.330719557195572e-06,
+      "loss": 1.3031,
+      "step": 1848
+    },
+    {
+      "epoch": 2.3011823273179837,
+      "grad_norm": 3.4802908897399902,
+      "learning_rate": 1.3284132841328414e-06,
+      "loss": 1.4001,
+      "step": 1849
+    },
+    {
+      "epoch": 2.302426882389546,
+      "grad_norm": 3.310987710952759,
+      "learning_rate": 1.3261070110701108e-06,
+      "loss": 1.3385,
+      "step": 1850
+    },
+    {
+      "epoch": 2.302426882389546,
+      "eval_loss": 1.615319848060608,
+      "eval_runtime": 44.8343,
+      "eval_samples_per_second": 22.304,
+      "eval_steps_per_second": 0.937,
+      "step": 1850
+    },
+    {
+      "epoch": 2.303671437461108,
+      "grad_norm": 3.3461661338806152,
+      "learning_rate": 1.3238007380073803e-06,
+      "loss": 1.3461,
+      "step": 1851
+    },
+    {
+      "epoch": 2.3049159925326697,
+      "grad_norm": 3.317770481109619,
+      "learning_rate": 1.3214944649446495e-06,
+      "loss": 1.338,
+      "step": 1852
+    },
+    {
+      "epoch": 2.3061605476042315,
+      "grad_norm": 3.3806679248809814,
+      "learning_rate": 1.319188191881919e-06,
+      "loss": 1.3367,
+      "step": 1853
+    },
+    {
+      "epoch": 2.3074051026757934,
+      "grad_norm": 3.366933822631836,
+      "learning_rate": 1.3168819188191883e-06,
+      "loss": 1.3717,
+      "step": 1854
+    },
+    {
+      "epoch": 2.308649657747355,
+      "grad_norm": 3.3250250816345215,
+      "learning_rate": 1.3145756457564575e-06,
+      "loss": 1.3585,
+      "step": 1855
+    },
+    {
+      "epoch": 2.309894212818917,
+      "grad_norm": 3.320171356201172,
+      "learning_rate": 1.3122693726937272e-06,
+      "loss": 1.3491,
+      "step": 1856
+    },
+    {
+      "epoch": 2.3111387678904793,
+      "grad_norm": 3.418029546737671,
+      "learning_rate": 1.3099630996309964e-06,
+      "loss": 1.3627,
+      "step": 1857
+    },
+    {
+      "epoch": 2.312383322962041,
+      "grad_norm": 3.3762004375457764,
+      "learning_rate": 1.3076568265682658e-06,
+      "loss": 1.3409,
+      "step": 1858
+    },
+    {
+      "epoch": 2.313627878033603,
+      "grad_norm": 3.3099405765533447,
+      "learning_rate": 1.3053505535055352e-06,
+      "loss": 1.3593,
+      "step": 1859
+    },
+    {
+      "epoch": 2.314872433105165,
+      "grad_norm": 3.4060535430908203,
+      "learning_rate": 1.3030442804428044e-06,
+      "loss": 1.3363,
+      "step": 1860
+    },
+    {
+      "epoch": 2.314872433105165,
+      "eval_loss": 1.6176154613494873,
+      "eval_runtime": 46.8022,
+      "eval_samples_per_second": 21.367,
+      "eval_steps_per_second": 0.897,
+      "step": 1860
+    },
+    {
+      "epoch": 2.3161169881767267,
+      "grad_norm": 3.2827699184417725,
+      "learning_rate": 1.3007380073800739e-06,
+      "loss": 1.2718,
+      "step": 1861
+    },
+    {
+      "epoch": 2.3173615432482886,
+      "grad_norm": 3.5225489139556885,
+      "learning_rate": 1.2984317343173433e-06,
+      "loss": 1.3728,
+      "step": 1862
+    },
+    {
+      "epoch": 2.318606098319851,
+      "grad_norm": 3.475665807723999,
+      "learning_rate": 1.2961254612546125e-06,
+      "loss": 1.3772,
+      "step": 1863
+    },
+    {
+      "epoch": 2.3198506533914127,
+      "grad_norm": 3.385239839553833,
+      "learning_rate": 1.2938191881918821e-06,
+      "loss": 1.3471,
+      "step": 1864
+    },
+    {
+      "epoch": 2.3210952084629746,
+      "grad_norm": 3.312469005584717,
+      "learning_rate": 1.2915129151291513e-06,
+      "loss": 1.2961,
+      "step": 1865
+    },
+    {
+      "epoch": 2.3223397635345364,
+      "grad_norm": 3.3883018493652344,
+      "learning_rate": 1.2892066420664208e-06,
+      "loss": 1.3778,
+      "step": 1866
+    },
+    {
+      "epoch": 2.3235843186060983,
+      "grad_norm": 3.368964195251465,
+      "learning_rate": 1.2869003690036902e-06,
+      "loss": 1.3153,
+      "step": 1867
+    },
+    {
+      "epoch": 2.32482887367766,
+      "grad_norm": 3.4132297039031982,
+      "learning_rate": 1.2845940959409594e-06,
+      "loss": 1.3718,
+      "step": 1868
+    },
+    {
+      "epoch": 2.326073428749222,
+      "grad_norm": 3.3750340938568115,
+      "learning_rate": 1.282287822878229e-06,
+      "loss": 1.4035,
+      "step": 1869
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "grad_norm": 3.372588872909546,
+      "learning_rate": 1.2799815498154982e-06,
+      "loss": 1.3591,
+      "step": 1870
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "eval_loss": 1.6196452379226685,
+      "eval_runtime": 43.8125,
+      "eval_samples_per_second": 22.825,
+      "eval_steps_per_second": 0.959,
+      "step": 1870
+    },
+    {
+      "epoch": 2.328562538892346,
+      "grad_norm": 3.341944694519043,
+      "learning_rate": 1.2776752767527675e-06,
+      "loss": 1.3239,
+      "step": 1871
+    },
+    {
+      "epoch": 2.329807093963908,
+      "grad_norm": 3.4855759143829346,
+      "learning_rate": 1.275369003690037e-06,
+      "loss": 1.3712,
+      "step": 1872
+    },
+    {
+      "epoch": 2.33105164903547,
+      "grad_norm": 3.577047348022461,
+      "learning_rate": 1.2730627306273063e-06,
+      "loss": 1.3502,
+      "step": 1873
+    },
+    {
+      "epoch": 2.3322962041070316,
+      "grad_norm": 3.479374885559082,
+      "learning_rate": 1.270756457564576e-06,
+      "loss": 1.3296,
+      "step": 1874
+    },
+    {
+      "epoch": 2.3335407591785935,
+      "grad_norm": 3.2203609943389893,
+      "learning_rate": 1.2684501845018451e-06,
+      "loss": 1.3105,
+      "step": 1875
+    },
+    {
+      "epoch": 2.334785314250156,
+      "grad_norm": 3.3177664279937744,
+      "learning_rate": 1.2661439114391144e-06,
+      "loss": 1.3365,
+      "step": 1876
+    },
+    {
+      "epoch": 2.3360298693217176,
+      "grad_norm": 3.224146842956543,
+      "learning_rate": 1.263837638376384e-06,
+      "loss": 1.3172,
+      "step": 1877
+    },
+    {
+      "epoch": 2.3372744243932795,
+      "grad_norm": 3.336106538772583,
+      "learning_rate": 1.2615313653136532e-06,
+      "loss": 1.3248,
+      "step": 1878
+    },
+    {
+      "epoch": 2.3385189794648413,
+      "grad_norm": 3.4414947032928467,
+      "learning_rate": 1.2592250922509224e-06,
+      "loss": 1.3637,
+      "step": 1879
+    },
+    {
+      "epoch": 2.339763534536403,
+      "grad_norm": 3.5528085231781006,
+      "learning_rate": 1.256918819188192e-06,
+      "loss": 1.3986,
+      "step": 1880
+    },
+    {
+      "epoch": 2.339763534536403,
+      "eval_loss": 1.6137934923171997,
+      "eval_runtime": 43.8859,
+      "eval_samples_per_second": 22.786,
+      "eval_steps_per_second": 0.957,
+      "step": 1880
+    },
+    {
+      "epoch": 2.341008089607965,
+      "grad_norm": 3.406446695327759,
+      "learning_rate": 1.2546125461254613e-06,
+      "loss": 1.3479,
+      "step": 1881
+    },
+    {
+      "epoch": 2.342252644679527,
+      "grad_norm": 3.4129297733306885,
+      "learning_rate": 1.2523062730627309e-06,
+      "loss": 1.3227,
+      "step": 1882
+    },
+    {
+      "epoch": 2.343497199751089,
+      "grad_norm": 3.3013577461242676,
+      "learning_rate": 1.25e-06,
+      "loss": 1.3277,
+      "step": 1883
+    },
+    {
+      "epoch": 2.344741754822651,
+      "grad_norm": 3.3039028644561768,
+      "learning_rate": 1.2476937269372695e-06,
+      "loss": 1.3208,
+      "step": 1884
+    },
+    {
+      "epoch": 2.345986309894213,
+      "grad_norm": 3.3239917755126953,
+      "learning_rate": 1.245387453874539e-06,
+      "loss": 1.3293,
+      "step": 1885
+    },
+    {
+      "epoch": 2.3472308649657747,
+      "grad_norm": 3.239821195602417,
+      "learning_rate": 1.2430811808118082e-06,
+      "loss": 1.3191,
+      "step": 1886
+    },
+    {
+      "epoch": 2.3484754200373366,
+      "grad_norm": 3.6869564056396484,
+      "learning_rate": 1.2407749077490776e-06,
+      "loss": 1.363,
+      "step": 1887
+    },
+    {
+      "epoch": 2.3497199751088984,
+      "grad_norm": 3.5775156021118164,
+      "learning_rate": 1.238468634686347e-06,
+      "loss": 1.3564,
+      "step": 1888
+    },
+    {
+      "epoch": 2.3509645301804607,
+      "grad_norm": 3.3177807331085205,
+      "learning_rate": 1.2361623616236164e-06,
+      "loss": 1.3044,
+      "step": 1889
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "grad_norm": 3.622527837753296,
+      "learning_rate": 1.2338560885608856e-06,
+      "loss": 1.381,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "eval_loss": 1.6165976524353027,
+      "eval_runtime": 44.4026,
+      "eval_samples_per_second": 22.521,
+      "eval_steps_per_second": 0.946,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3534536403235844,
+      "grad_norm": 3.3163886070251465,
+      "learning_rate": 1.231549815498155e-06,
+      "loss": 1.3598,
+      "step": 1891
+    },
+    {
+      "epoch": 2.3546981953951462,
+      "grad_norm": 3.3344099521636963,
+      "learning_rate": 1.2292435424354245e-06,
+      "loss": 1.3733,
+      "step": 1892
+    },
+    {
+      "epoch": 2.355942750466708,
+      "grad_norm": 3.259413480758667,
+      "learning_rate": 1.226937269372694e-06,
+      "loss": 1.3281,
+      "step": 1893
+    },
+    {
+      "epoch": 2.35718730553827,
+      "grad_norm": 3.427147150039673,
+      "learning_rate": 1.2246309963099633e-06,
+      "loss": 1.3484,
+      "step": 1894
+    },
+    {
+      "epoch": 2.358431860609832,
+      "grad_norm": 3.3890154361724854,
+      "learning_rate": 1.2223247232472325e-06,
+      "loss": 1.3768,
+      "step": 1895
+    },
+    {
+      "epoch": 2.359676415681394,
+      "grad_norm": 3.2845499515533447,
+      "learning_rate": 1.220018450184502e-06,
+      "loss": 1.3579,
+      "step": 1896
+    },
+    {
+      "epoch": 2.360920970752956,
+      "grad_norm": 3.3046178817749023,
+      "learning_rate": 1.2177121771217714e-06,
+      "loss": 1.3322,
+      "step": 1897
+    },
+    {
+      "epoch": 2.3621655258245178,
+      "grad_norm": 3.4520273208618164,
+      "learning_rate": 1.2154059040590408e-06,
+      "loss": 1.3515,
+      "step": 1898
+    },
+    {
+      "epoch": 2.3634100808960796,
+      "grad_norm": 3.3665220737457275,
+      "learning_rate": 1.21309963099631e-06,
+      "loss": 1.3236,
+      "step": 1899
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "grad_norm": 3.3551578521728516,
+      "learning_rate": 1.2107933579335794e-06,
+      "loss": 1.3568,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "eval_loss": 1.6159367561340332,
+      "eval_runtime": 44.1216,
+      "eval_samples_per_second": 22.665,
+      "eval_steps_per_second": 0.952,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3658991910392033,
+      "grad_norm": 3.6204397678375244,
+      "learning_rate": 1.2084870848708489e-06,
+      "loss": 1.4072,
+      "step": 1901
+    },
+    {
+      "epoch": 2.3671437461107656,
+      "grad_norm": 3.3525922298431396,
+      "learning_rate": 1.2061808118081183e-06,
+      "loss": 1.328,
+      "step": 1902
+    },
+    {
+      "epoch": 2.3683883011823275,
+      "grad_norm": 3.3686022758483887,
+      "learning_rate": 1.2038745387453875e-06,
+      "loss": 1.3086,
+      "step": 1903
+    },
+    {
+      "epoch": 2.3696328562538893,
+      "grad_norm": 3.5756561756134033,
+      "learning_rate": 1.201568265682657e-06,
+      "loss": 1.3784,
+      "step": 1904
+    },
+    {
+      "epoch": 2.370877411325451,
+      "grad_norm": 3.4186534881591797,
+      "learning_rate": 1.1992619926199263e-06,
+      "loss": 1.3546,
+      "step": 1905
+    },
+    {
+      "epoch": 2.372121966397013,
+      "grad_norm": 3.367868661880493,
+      "learning_rate": 1.1969557195571958e-06,
+      "loss": 1.3527,
+      "step": 1906
+    },
+    {
+      "epoch": 2.373366521468575,
+      "grad_norm": 3.3395912647247314,
+      "learning_rate": 1.1946494464944652e-06,
+      "loss": 1.3366,
+      "step": 1907
+    },
+    {
+      "epoch": 2.3746110765401367,
+      "grad_norm": 3.2967982292175293,
+      "learning_rate": 1.1923431734317344e-06,
+      "loss": 1.3229,
+      "step": 1908
+    },
+    {
+      "epoch": 2.375855631611699,
+      "grad_norm": 3.2422475814819336,
+      "learning_rate": 1.1900369003690038e-06,
+      "loss": 1.3455,
+      "step": 1909
+    },
+    {
+      "epoch": 2.377100186683261,
+      "grad_norm": 3.3781206607818604,
+      "learning_rate": 1.1877306273062733e-06,
+      "loss": 1.3314,
+      "step": 1910
+    },
+    {
+      "epoch": 2.377100186683261,
+      "eval_loss": 1.6179407835006714,
+      "eval_runtime": 42.9089,
+      "eval_samples_per_second": 23.305,
+      "eval_steps_per_second": 0.979,
+      "step": 1910
+    },
+    {
+      "epoch": 2.3783447417548227,
+      "grad_norm": 3.438586711883545,
+      "learning_rate": 1.1854243542435427e-06,
+      "loss": 1.3654,
+      "step": 1911
+    },
+    {
+      "epoch": 2.3795892968263845,
+      "grad_norm": 3.503896474838257,
+      "learning_rate": 1.1831180811808119e-06,
+      "loss": 1.3716,
+      "step": 1912
+    },
+    {
+      "epoch": 2.3808338518979464,
+      "grad_norm": 3.411320686340332,
+      "learning_rate": 1.1808118081180813e-06,
+      "loss": 1.3315,
+      "step": 1913
+    },
+    {
+      "epoch": 2.3820784069695082,
+      "grad_norm": 3.1860697269439697,
+      "learning_rate": 1.1785055350553507e-06,
+      "loss": 1.3016,
+      "step": 1914
+    },
+    {
+      "epoch": 2.3833229620410705,
+      "grad_norm": 3.4494664669036865,
+      "learning_rate": 1.1761992619926202e-06,
+      "loss": 1.3629,
+      "step": 1915
+    },
+    {
+      "epoch": 2.3845675171126324,
+      "grad_norm": 3.303277015686035,
+      "learning_rate": 1.1738929889298894e-06,
+      "loss": 1.3207,
+      "step": 1916
+    },
+    {
+      "epoch": 2.385812072184194,
+      "grad_norm": 3.254411458969116,
+      "learning_rate": 1.1715867158671588e-06,
+      "loss": 1.3477,
+      "step": 1917
+    },
+    {
+      "epoch": 2.387056627255756,
+      "grad_norm": 3.258666753768921,
+      "learning_rate": 1.1692804428044282e-06,
+      "loss": 1.3452,
+      "step": 1918
+    },
+    {
+      "epoch": 2.388301182327318,
+      "grad_norm": 3.3814001083374023,
+      "learning_rate": 1.1669741697416974e-06,
+      "loss": 1.329,
+      "step": 1919
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "grad_norm": 3.367011547088623,
+      "learning_rate": 1.1646678966789668e-06,
+      "loss": 1.3326,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "eval_loss": 1.6158324480056763,
+      "eval_runtime": 45.8088,
+      "eval_samples_per_second": 21.83,
+      "eval_steps_per_second": 0.917,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3907902924704416,
+      "grad_norm": 3.313123941421509,
+      "learning_rate": 1.1623616236162363e-06,
+      "loss": 1.307,
+      "step": 1921
+    },
+    {
+      "epoch": 2.392034847542004,
+      "grad_norm": 3.27423357963562,
+      "learning_rate": 1.1600553505535057e-06,
+      "loss": 1.3249,
+      "step": 1922
+    },
+    {
+      "epoch": 2.3932794026135658,
+      "grad_norm": 3.2963404655456543,
+      "learning_rate": 1.157749077490775e-06,
+      "loss": 1.3394,
+      "step": 1923
+    },
+    {
+      "epoch": 2.3945239576851276,
+      "grad_norm": 3.387673854827881,
+      "learning_rate": 1.1554428044280443e-06,
+      "loss": 1.3586,
+      "step": 1924
+    },
+    {
+      "epoch": 2.3957685127566895,
+      "grad_norm": 3.2588143348693848,
+      "learning_rate": 1.1531365313653137e-06,
+      "loss": 1.317,
+      "step": 1925
+    },
+    {
+      "epoch": 2.3970130678282513,
+      "grad_norm": 3.285653829574585,
+      "learning_rate": 1.1508302583025832e-06,
+      "loss": 1.2847,
+      "step": 1926
+    },
+    {
+      "epoch": 2.398257622899813,
+      "grad_norm": 3.4495935440063477,
+      "learning_rate": 1.1485239852398524e-06,
+      "loss": 1.3815,
+      "step": 1927
+    },
+    {
+      "epoch": 2.3995021779713754,
+      "grad_norm": 3.287186622619629,
+      "learning_rate": 1.1462177121771218e-06,
+      "loss": 1.3048,
+      "step": 1928
+    },
+    {
+      "epoch": 2.4007467330429373,
+      "grad_norm": 3.318042278289795,
+      "learning_rate": 1.1439114391143912e-06,
+      "loss": 1.313,
+      "step": 1929
+    },
+    {
+      "epoch": 2.401991288114499,
+      "grad_norm": 3.2705087661743164,
+      "learning_rate": 1.1416051660516606e-06,
+      "loss": 1.3207,
+      "step": 1930
+    },
+    {
+      "epoch": 2.401991288114499,
+      "eval_loss": 1.613088846206665,
+      "eval_runtime": 46.0901,
+      "eval_samples_per_second": 21.697,
+      "eval_steps_per_second": 0.911,
+      "step": 1930
+    },
+    {
+      "epoch": 2.403235843186061,
+      "grad_norm": 3.307523250579834,
+      "learning_rate": 1.1392988929889299e-06,
+      "loss": 1.3312,
+      "step": 1931
+    },
+    {
+      "epoch": 2.404480398257623,
+      "grad_norm": 3.3040900230407715,
+      "learning_rate": 1.1369926199261993e-06,
+      "loss": 1.3279,
+      "step": 1932
+    },
+    {
+      "epoch": 2.4057249533291847,
+      "grad_norm": 3.232649087905884,
+      "learning_rate": 1.1346863468634687e-06,
+      "loss": 1.314,
+      "step": 1933
+    },
+    {
+      "epoch": 2.4069695084007465,
+      "grad_norm": 3.3581860065460205,
+      "learning_rate": 1.1323800738007381e-06,
+      "loss": 1.3158,
+      "step": 1934
+    },
+    {
+      "epoch": 2.408214063472309,
+      "grad_norm": 3.378774404525757,
+      "learning_rate": 1.1300738007380073e-06,
+      "loss": 1.3252,
+      "step": 1935
+    },
+    {
+      "epoch": 2.4094586185438707,
+      "grad_norm": 3.4180240631103516,
+      "learning_rate": 1.1277675276752768e-06,
+      "loss": 1.3606,
+      "step": 1936
+    },
+    {
+      "epoch": 2.4107031736154325,
+      "grad_norm": 3.410921335220337,
+      "learning_rate": 1.1254612546125462e-06,
+      "loss": 1.3484,
+      "step": 1937
+    },
+    {
+      "epoch": 2.4119477286869944,
+      "grad_norm": 3.4201443195343018,
+      "learning_rate": 1.1231549815498156e-06,
+      "loss": 1.3659,
+      "step": 1938
+    },
+    {
+      "epoch": 2.413192283758556,
+      "grad_norm": 3.381019115447998,
+      "learning_rate": 1.1208487084870848e-06,
+      "loss": 1.3261,
+      "step": 1939
+    },
+    {
+      "epoch": 2.414436838830118,
+      "grad_norm": 3.516425371170044,
+      "learning_rate": 1.1185424354243542e-06,
+      "loss": 1.3618,
+      "step": 1940
+    },
+    {
+      "epoch": 2.414436838830118,
+      "eval_loss": 1.6142985820770264,
+      "eval_runtime": 45.8868,
+      "eval_samples_per_second": 21.793,
+      "eval_steps_per_second": 0.915,
+      "step": 1940
+    },
+    {
+      "epoch": 2.4156813939016804,
+      "grad_norm": 3.5319905281066895,
+      "learning_rate": 1.1162361623616237e-06,
+      "loss": 1.338,
+      "step": 1941
+    },
+    {
+      "epoch": 2.416925948973242,
+      "grad_norm": 3.4355592727661133,
+      "learning_rate": 1.113929889298893e-06,
+      "loss": 1.3389,
+      "step": 1942
+    },
+    {
+      "epoch": 2.418170504044804,
+      "grad_norm": 3.362187623977661,
+      "learning_rate": 1.1116236162361623e-06,
+      "loss": 1.3194,
+      "step": 1943
+    },
+    {
+      "epoch": 2.419415059116366,
+      "grad_norm": 3.4991087913513184,
+      "learning_rate": 1.1093173431734317e-06,
+      "loss": 1.3758,
+      "step": 1944
+    },
+    {
+      "epoch": 2.4206596141879277,
+      "grad_norm": 3.5951290130615234,
+      "learning_rate": 1.1070110701107011e-06,
+      "loss": 1.3548,
+      "step": 1945
+    },
+    {
+      "epoch": 2.4219041692594896,
+      "grad_norm": 3.3458032608032227,
+      "learning_rate": 1.1047047970479706e-06,
+      "loss": 1.3483,
+      "step": 1946
+    },
+    {
+      "epoch": 2.4231487243310514,
+      "grad_norm": 3.426600933074951,
+      "learning_rate": 1.10239852398524e-06,
+      "loss": 1.3548,
+      "step": 1947
+    },
+    {
+      "epoch": 2.4243932794026137,
+      "grad_norm": 3.5074410438537598,
+      "learning_rate": 1.1000922509225092e-06,
+      "loss": 1.3608,
+      "step": 1948
+    },
+    {
+      "epoch": 2.4256378344741756,
+      "grad_norm": 3.5622129440307617,
+      "learning_rate": 1.0977859778597786e-06,
+      "loss": 1.3861,
+      "step": 1949
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "grad_norm": 3.537771463394165,
+      "learning_rate": 1.095479704797048e-06,
+      "loss": 1.3408,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "eval_loss": 1.612324833869934,
+      "eval_runtime": 46.7327,
+      "eval_samples_per_second": 21.398,
+      "eval_steps_per_second": 0.899,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4281269446172993,
+      "grad_norm": 3.430544137954712,
+      "learning_rate": 1.0931734317343175e-06,
+      "loss": 1.3678,
+      "step": 1951
+    },
+    {
+      "epoch": 2.429371499688861,
+      "grad_norm": 3.2243361473083496,
+      "learning_rate": 1.0908671586715867e-06,
+      "loss": 1.2982,
+      "step": 1952
+    },
+    {
+      "epoch": 2.430616054760423,
+      "grad_norm": 3.426236867904663,
+      "learning_rate": 1.088560885608856e-06,
+      "loss": 1.3827,
+      "step": 1953
+    },
+    {
+      "epoch": 2.4318606098319853,
+      "grad_norm": 3.3180603981018066,
+      "learning_rate": 1.0862546125461255e-06,
+      "loss": 1.3065,
+      "step": 1954
+    },
+    {
+      "epoch": 2.433105164903547,
+      "grad_norm": 3.3775084018707275,
+      "learning_rate": 1.083948339483395e-06,
+      "loss": 1.2759,
+      "step": 1955
+    },
+    {
+      "epoch": 2.434349719975109,
+      "grad_norm": 3.4124209880828857,
+      "learning_rate": 1.0816420664206644e-06,
+      "loss": 1.3571,
+      "step": 1956
+    },
+    {
+      "epoch": 2.435594275046671,
+      "grad_norm": 3.494408369064331,
+      "learning_rate": 1.0793357933579336e-06,
+      "loss": 1.3931,
+      "step": 1957
+    },
+    {
+      "epoch": 2.4368388301182327,
+      "grad_norm": 3.2699358463287354,
+      "learning_rate": 1.077029520295203e-06,
+      "loss": 1.3182,
+      "step": 1958
+    },
+    {
+      "epoch": 2.4380833851897945,
+      "grad_norm": 3.281592607498169,
+      "learning_rate": 1.0747232472324724e-06,
+      "loss": 1.2629,
+      "step": 1959
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "grad_norm": 3.3045406341552734,
+      "learning_rate": 1.0724169741697419e-06,
+      "loss": 1.3093,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "eval_loss": 1.6135376691818237,
+      "eval_runtime": 51.0587,
+      "eval_samples_per_second": 19.585,
+      "eval_steps_per_second": 0.823,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4405724953329186,
+      "grad_norm": 3.3883392810821533,
+      "learning_rate": 1.070110701107011e-06,
+      "loss": 1.4005,
+      "step": 1961
+    },
+    {
+      "epoch": 2.4418170504044805,
+      "grad_norm": 3.3957419395446777,
+      "learning_rate": 1.0678044280442805e-06,
+      "loss": 1.3342,
+      "step": 1962
+    },
+    {
+      "epoch": 2.4430616054760423,
+      "grad_norm": 3.1982152462005615,
+      "learning_rate": 1.06549815498155e-06,
+      "loss": 1.3273,
+      "step": 1963
+    },
+    {
+      "epoch": 2.444306160547604,
+      "grad_norm": 3.3369827270507812,
+      "learning_rate": 1.0631918819188193e-06,
+      "loss": 1.312,
+      "step": 1964
+    },
+    {
+      "epoch": 2.445550715619166,
+      "grad_norm": 3.3651299476623535,
+      "learning_rate": 1.0608856088560888e-06,
+      "loss": 1.3728,
+      "step": 1965
+    },
+    {
+      "epoch": 2.446795270690728,
+      "grad_norm": 3.3393259048461914,
+      "learning_rate": 1.058579335793358e-06,
+      "loss": 1.3454,
+      "step": 1966
+    },
+    {
+      "epoch": 2.44803982576229,
+      "grad_norm": 3.4974067211151123,
+      "learning_rate": 1.0562730627306274e-06,
+      "loss": 1.3747,
+      "step": 1967
+    },
+    {
+      "epoch": 2.449284380833852,
+      "grad_norm": 3.4064528942108154,
+      "learning_rate": 1.0539667896678968e-06,
+      "loss": 1.3695,
+      "step": 1968
+    },
+    {
+      "epoch": 2.450528935905414,
+      "grad_norm": 3.347919225692749,
+      "learning_rate": 1.0516605166051662e-06,
+      "loss": 1.3439,
+      "step": 1969
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "grad_norm": 3.4505932331085205,
+      "learning_rate": 1.0493542435424354e-06,
+      "loss": 1.3723,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "eval_loss": 1.611203908920288,
+      "eval_runtime": 52.5131,
+      "eval_samples_per_second": 19.043,
+      "eval_steps_per_second": 0.8,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4530180460485376,
+      "grad_norm": 3.4287805557250977,
+      "learning_rate": 1.0470479704797049e-06,
+      "loss": 1.3426,
+      "step": 1971
+    },
+    {
+      "epoch": 2.4542626011200994,
+      "grad_norm": 3.4337522983551025,
+      "learning_rate": 1.0447416974169743e-06,
+      "loss": 1.3451,
+      "step": 1972
+    },
+    {
+      "epoch": 2.4555071561916613,
+      "grad_norm": 3.4307212829589844,
+      "learning_rate": 1.0424354243542437e-06,
+      "loss": 1.3097,
+      "step": 1973
+    },
+    {
+      "epoch": 2.4567517112632236,
+      "grad_norm": 3.307767868041992,
+      "learning_rate": 1.0401291512915131e-06,
+      "loss": 1.2751,
+      "step": 1974
+    },
+    {
+      "epoch": 2.4579962663347854,
+      "grad_norm": 3.4809703826904297,
+      "learning_rate": 1.0378228782287824e-06,
+      "loss": 1.3522,
+      "step": 1975
+    },
+    {
+      "epoch": 2.4592408214063473,
+      "grad_norm": 3.235170602798462,
+      "learning_rate": 1.0355166051660518e-06,
+      "loss": 1.3003,
+      "step": 1976
+    },
+    {
+      "epoch": 2.460485376477909,
+      "grad_norm": 3.2969000339508057,
+      "learning_rate": 1.0332103321033212e-06,
+      "loss": 1.2993,
+      "step": 1977
+    },
+    {
+      "epoch": 2.461729931549471,
+      "grad_norm": 3.311535358428955,
+      "learning_rate": 1.0309040590405906e-06,
+      "loss": 1.3262,
+      "step": 1978
+    },
+    {
+      "epoch": 2.462974486621033,
+      "grad_norm": 3.2320244312286377,
+      "learning_rate": 1.0285977859778598e-06,
+      "loss": 1.3111,
+      "step": 1979
+    },
+    {
+      "epoch": 2.464219041692595,
+      "grad_norm": 3.2673771381378174,
+      "learning_rate": 1.0262915129151293e-06,
+      "loss": 1.3439,
+      "step": 1980
+    },
+    {
+      "epoch": 2.464219041692595,
+      "eval_loss": 1.6103569269180298,
+      "eval_runtime": 47.2563,
+      "eval_samples_per_second": 21.161,
+      "eval_steps_per_second": 0.889,
+      "step": 1980
+    },
+    {
+      "epoch": 2.465463596764157,
+      "grad_norm": 3.324113130569458,
+      "learning_rate": 1.0239852398523987e-06,
+      "loss": 1.3278,
+      "step": 1981
+    },
+    {
+      "epoch": 2.466708151835719,
+      "grad_norm": 3.3066444396972656,
+      "learning_rate": 1.021678966789668e-06,
+      "loss": 1.3438,
+      "step": 1982
+    },
+    {
+      "epoch": 2.4679527069072806,
+      "grad_norm": 3.463088035583496,
+      "learning_rate": 1.0193726937269373e-06,
+      "loss": 1.3293,
+      "step": 1983
+    },
+    {
+      "epoch": 2.4691972619788425,
+      "grad_norm": 3.406008005142212,
+      "learning_rate": 1.0170664206642067e-06,
+      "loss": 1.4026,
+      "step": 1984
+    },
+    {
+      "epoch": 2.4704418170504043,
+      "grad_norm": 3.408823251724243,
+      "learning_rate": 1.0147601476014762e-06,
+      "loss": 1.3226,
+      "step": 1985
+    },
+    {
+      "epoch": 2.471686372121966,
+      "grad_norm": 3.2991373538970947,
+      "learning_rate": 1.0124538745387456e-06,
+      "loss": 1.2682,
+      "step": 1986
+    },
+    {
+      "epoch": 2.4729309271935285,
+      "grad_norm": 3.3868234157562256,
+      "learning_rate": 1.010147601476015e-06,
+      "loss": 1.3286,
+      "step": 1987
+    },
+    {
+      "epoch": 2.4741754822650903,
+      "grad_norm": 3.3641555309295654,
+      "learning_rate": 1.0078413284132842e-06,
+      "loss": 1.3423,
+      "step": 1988
+    },
+    {
+      "epoch": 2.475420037336652,
+      "grad_norm": 3.3237619400024414,
+      "learning_rate": 1.0055350553505536e-06,
+      "loss": 1.2979,
+      "step": 1989
+    },
+    {
+      "epoch": 2.476664592408214,
+      "grad_norm": 3.6278553009033203,
+      "learning_rate": 1.003228782287823e-06,
+      "loss": 1.3543,
+      "step": 1990
+    },
+    {
+      "epoch": 2.476664592408214,
+      "eval_loss": 1.6103031635284424,
+      "eval_runtime": 46.0573,
+      "eval_samples_per_second": 21.712,
+      "eval_steps_per_second": 0.912,
+      "step": 1990
+    },
+    {
+      "epoch": 2.477909147479776,
+      "grad_norm": 3.2861995697021484,
+      "learning_rate": 1.0009225092250923e-06,
+      "loss": 1.3303,
+      "step": 1991
+    },
+    {
+      "epoch": 2.4791537025513377,
+      "grad_norm": 3.352261781692505,
+      "learning_rate": 9.986162361623617e-07,
+      "loss": 1.3527,
+      "step": 1992
+    },
+    {
+      "epoch": 2.4803982576229,
+      "grad_norm": 3.483726978302002,
+      "learning_rate": 9.963099630996311e-07,
+      "loss": 1.3762,
+      "step": 1993
+    },
+    {
+      "epoch": 2.481642812694462,
+      "grad_norm": 3.357154607772827,
+      "learning_rate": 9.940036900369005e-07,
+      "loss": 1.3601,
+      "step": 1994
+    },
+    {
+      "epoch": 2.4828873677660237,
+      "grad_norm": 3.4477577209472656,
+      "learning_rate": 9.916974169741698e-07,
+      "loss": 1.3636,
+      "step": 1995
+    },
+    {
+      "epoch": 2.4841319228375855,
+      "grad_norm": 3.377279043197632,
+      "learning_rate": 9.893911439114392e-07,
+      "loss": 1.3349,
+      "step": 1996
+    },
+    {
+      "epoch": 2.4853764779091474,
+      "grad_norm": 3.2705130577087402,
+      "learning_rate": 9.870848708487086e-07,
+      "loss": 1.3659,
+      "step": 1997
+    },
+    {
+      "epoch": 2.4866210329807092,
+      "grad_norm": 3.418225049972534,
+      "learning_rate": 9.84778597785978e-07,
+      "loss": 1.3534,
+      "step": 1998
+    },
+    {
+      "epoch": 2.487865588052271,
+      "grad_norm": 3.2288014888763428,
+      "learning_rate": 9.824723247232472e-07,
+      "loss": 1.3392,
+      "step": 1999
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "grad_norm": 3.3513882160186768,
+      "learning_rate": 9.801660516605167e-07,
+      "loss": 1.3483,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "eval_loss": 1.6108943223953247,
+      "eval_runtime": 48.275,
+      "eval_samples_per_second": 20.715,
+      "eval_steps_per_second": 0.87,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4903546981953952,
+      "grad_norm": 3.35583758354187,
+      "learning_rate": 9.77859778597786e-07,
+      "loss": 1.3808,
+      "step": 2001
+    },
+    {
+      "epoch": 2.491599253266957,
+      "grad_norm": 3.4611411094665527,
+      "learning_rate": 9.755535055350555e-07,
+      "loss": 1.3512,
+      "step": 2002
+    },
+    {
+      "epoch": 2.492843808338519,
+      "grad_norm": 3.389965057373047,
+      "learning_rate": 9.732472324723247e-07,
+      "loss": 1.2777,
+      "step": 2003
+    },
+    {
+      "epoch": 2.4940883634100808,
+      "grad_norm": 3.370044231414795,
+      "learning_rate": 9.709409594095941e-07,
+      "loss": 1.3407,
+      "step": 2004
+    },
+    {
+      "epoch": 2.4953329184816426,
+      "grad_norm": 3.4178175926208496,
+      "learning_rate": 9.686346863468636e-07,
+      "loss": 1.3419,
+      "step": 2005
+    },
+    {
+      "epoch": 2.496577473553205,
+      "grad_norm": 3.4306576251983643,
+      "learning_rate": 9.66328413284133e-07,
+      "loss": 1.3029,
+      "step": 2006
+    },
+    {
+      "epoch": 2.4978220286247668,
+      "grad_norm": 3.4822888374328613,
+      "learning_rate": 9.640221402214022e-07,
+      "loss": 1.3512,
+      "step": 2007
+    },
+    {
+      "epoch": 2.4990665836963286,
+      "grad_norm": 3.2686867713928223,
+      "learning_rate": 9.617158671586716e-07,
+      "loss": 1.3366,
+      "step": 2008
+    },
+    {
+      "epoch": 2.5003111387678905,
+      "grad_norm": 3.413224220275879,
+      "learning_rate": 9.59409594095941e-07,
+      "loss": 1.324,
+      "step": 2009
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "grad_norm": 3.180180311203003,
+      "learning_rate": 9.571033210332105e-07,
+      "loss": 1.3054,
+      "step": 2010
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "eval_loss": 1.6145573854446411,
+      "eval_runtime": 42.6598,
+      "eval_samples_per_second": 23.441,
+      "eval_steps_per_second": 0.985,
+      "step": 2010
+    },
+    {
+      "epoch": 2.502800248911014,
+      "grad_norm": 3.4968016147613525,
+      "learning_rate": 9.547970479704797e-07,
+      "loss": 1.3782,
+      "step": 2011
+    },
+    {
+      "epoch": 2.504044803982576,
+      "grad_norm": 3.270414113998413,
+      "learning_rate": 9.524907749077492e-07,
+      "loss": 1.3389,
+      "step": 2012
+    },
+    {
+      "epoch": 2.5052893590541383,
+      "grad_norm": 3.3850443363189697,
+      "learning_rate": 9.501845018450185e-07,
+      "loss": 1.3335,
+      "step": 2013
+    },
+    {
+      "epoch": 2.5065339141257,
+      "grad_norm": 3.3836278915405273,
+      "learning_rate": 9.478782287822879e-07,
+      "loss": 1.3343,
+      "step": 2014
+    },
+    {
+      "epoch": 2.507778469197262,
+      "grad_norm": 3.4950475692749023,
+      "learning_rate": 9.455719557195573e-07,
+      "loss": 1.337,
+      "step": 2015
+    },
+    {
+      "epoch": 2.509023024268824,
+      "grad_norm": 3.368262529373169,
+      "learning_rate": 9.432656826568267e-07,
+      "loss": 1.32,
+      "step": 2016
+    },
+    {
+      "epoch": 2.5102675793403857,
+      "grad_norm": 3.4199533462524414,
+      "learning_rate": 9.40959409594096e-07,
+      "loss": 1.3754,
+      "step": 2017
+    },
+    {
+      "epoch": 2.511512134411948,
+      "grad_norm": 3.3372180461883545,
+      "learning_rate": 9.386531365313654e-07,
+      "loss": 1.3319,
+      "step": 2018
+    },
+    {
+      "epoch": 2.51275668948351,
+      "grad_norm": 3.3614137172698975,
+      "learning_rate": 9.363468634686347e-07,
+      "loss": 1.383,
+      "step": 2019
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "grad_norm": 3.4617390632629395,
+      "learning_rate": 9.340405904059042e-07,
+      "loss": 1.3628,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "eval_loss": 1.6153606176376343,
+      "eval_runtime": 46.8889,
+      "eval_samples_per_second": 21.327,
+      "eval_steps_per_second": 0.896,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5152457996266335,
+      "grad_norm": 3.3711016178131104,
+      "learning_rate": 9.317343173431735e-07,
+      "loss": 1.3449,
+      "step": 2021
+    },
+    {
+      "epoch": 2.5164903546981954,
+      "grad_norm": 3.2858471870422363,
+      "learning_rate": 9.294280442804429e-07,
+      "loss": 1.348,
+      "step": 2022
+    },
+    {
+      "epoch": 2.5177349097697572,
+      "grad_norm": 3.445166826248169,
+      "learning_rate": 9.271217712177122e-07,
+      "loss": 1.3182,
+      "step": 2023
+    },
+    {
+      "epoch": 2.518979464841319,
+      "grad_norm": 3.417206287384033,
+      "learning_rate": 9.248154981549816e-07,
+      "loss": 1.3241,
+      "step": 2024
+    },
+    {
+      "epoch": 2.520224019912881,
+      "grad_norm": 3.333965539932251,
+      "learning_rate": 9.22509225092251e-07,
+      "loss": 1.3214,
+      "step": 2025
+    },
+    {
+      "epoch": 2.521468574984443,
+      "grad_norm": 3.4090497493743896,
+      "learning_rate": 9.202029520295204e-07,
+      "loss": 1.3566,
+      "step": 2026
+    },
+    {
+      "epoch": 2.522713130056005,
+      "grad_norm": 3.312790870666504,
+      "learning_rate": 9.178966789667898e-07,
+      "loss": 1.3181,
+      "step": 2027
+    },
+    {
+      "epoch": 2.523957685127567,
+      "grad_norm": 3.254122734069824,
+      "learning_rate": 9.155904059040591e-07,
+      "loss": 1.3079,
+      "step": 2028
+    },
+    {
+      "epoch": 2.5252022401991288,
+      "grad_norm": 3.5144238471984863,
+      "learning_rate": 9.132841328413284e-07,
+      "loss": 1.3586,
+      "step": 2029
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "grad_norm": 3.6083173751831055,
+      "learning_rate": 9.109778597785979e-07,
+      "loss": 1.3833,
+      "step": 2030
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "eval_loss": 1.6153796911239624,
+      "eval_runtime": 52.1312,
+      "eval_samples_per_second": 19.182,
+      "eval_steps_per_second": 0.806,
+      "step": 2030
+    },
+    {
+      "epoch": 2.527691350342253,
+      "grad_norm": 3.489710569381714,
+      "learning_rate": 9.086715867158673e-07,
+      "loss": 1.3466,
+      "step": 2031
+    },
+    {
+      "epoch": 2.5289359054138147,
+      "grad_norm": 3.4363298416137695,
+      "learning_rate": 9.063653136531365e-07,
+      "loss": 1.3461,
+      "step": 2032
+    },
+    {
+      "epoch": 2.5301804604853766,
+      "grad_norm": 3.627476930618286,
+      "learning_rate": 9.040590405904059e-07,
+      "loss": 1.3655,
+      "step": 2033
+    },
+    {
+      "epoch": 2.5314250155569384,
+      "grad_norm": 3.2995362281799316,
+      "learning_rate": 9.017527675276753e-07,
+      "loss": 1.297,
+      "step": 2034
+    },
+    {
+      "epoch": 2.5326695706285003,
+      "grad_norm": 3.4116721153259277,
+      "learning_rate": 8.994464944649448e-07,
+      "loss": 1.3319,
+      "step": 2035
+    },
+    {
+      "epoch": 2.533914125700062,
+      "grad_norm": 3.4598546028137207,
+      "learning_rate": 8.971402214022142e-07,
+      "loss": 1.3646,
+      "step": 2036
+    },
+    {
+      "epoch": 2.535158680771624,
+      "grad_norm": 3.4111695289611816,
+      "learning_rate": 8.948339483394834e-07,
+      "loss": 1.322,
+      "step": 2037
+    },
+    {
+      "epoch": 2.536403235843186,
+      "grad_norm": 3.371891736984253,
+      "learning_rate": 8.925276752767528e-07,
+      "loss": 1.362,
+      "step": 2038
+    },
+    {
+      "epoch": 2.537647790914748,
+      "grad_norm": 3.4267642498016357,
+      "learning_rate": 8.902214022140222e-07,
+      "loss": 1.3829,
+      "step": 2039
+    },
+    {
+      "epoch": 2.53889234598631,
+      "grad_norm": 3.2384777069091797,
+      "learning_rate": 8.879151291512917e-07,
+      "loss": 1.2771,
+      "step": 2040
+    },
+    {
+      "epoch": 2.53889234598631,
+      "eval_loss": 1.6134564876556396,
+      "eval_runtime": 49.4544,
+      "eval_samples_per_second": 20.221,
+      "eval_steps_per_second": 0.849,
+      "step": 2040
+    },
+    {
+      "epoch": 2.540136901057872,
+      "grad_norm": 3.3554086685180664,
+      "learning_rate": 8.856088560885609e-07,
+      "loss": 1.327,
+      "step": 2041
+    },
+    {
+      "epoch": 2.5413814561294337,
+      "grad_norm": 3.297962188720703,
+      "learning_rate": 8.833025830258303e-07,
+      "loss": 1.3257,
+      "step": 2042
+    },
+    {
+      "epoch": 2.5426260112009955,
+      "grad_norm": 3.4998221397399902,
+      "learning_rate": 8.809963099630997e-07,
+      "loss": 1.3782,
+      "step": 2043
+    },
+    {
+      "epoch": 2.543870566272558,
+      "grad_norm": 3.3661816120147705,
+      "learning_rate": 8.786900369003691e-07,
+      "loss": 1.3338,
+      "step": 2044
+    },
+    {
+      "epoch": 2.5451151213441197,
+      "grad_norm": 3.7328436374664307,
+      "learning_rate": 8.763837638376385e-07,
+      "loss": 1.4149,
+      "step": 2045
+    },
+    {
+      "epoch": 2.5463596764156815,
+      "grad_norm": 3.621159315109253,
+      "learning_rate": 8.740774907749078e-07,
+      "loss": 1.3104,
+      "step": 2046
+    },
+    {
+      "epoch": 2.5476042314872434,
+      "grad_norm": 3.5403215885162354,
+      "learning_rate": 8.717712177121772e-07,
+      "loss": 1.3278,
+      "step": 2047
+    },
+    {
+      "epoch": 2.548848786558805,
+      "grad_norm": 3.3124406337738037,
+      "learning_rate": 8.694649446494466e-07,
+      "loss": 1.3313,
+      "step": 2048
+    },
+    {
+      "epoch": 2.550093341630367,
+      "grad_norm": 3.4480574131011963,
+      "learning_rate": 8.671586715867159e-07,
+      "loss": 1.3225,
+      "step": 2049
+    },
+    {
+      "epoch": 2.551337896701929,
+      "grad_norm": 3.372326135635376,
+      "learning_rate": 8.648523985239853e-07,
+      "loss": 1.3377,
+      "step": 2050
+    },
+    {
+      "epoch": 2.551337896701929,
+      "eval_loss": 1.6056281328201294,
+      "eval_runtime": 52.0326,
+      "eval_samples_per_second": 19.219,
+      "eval_steps_per_second": 0.807,
+      "step": 2050
+    },
+    {
+      "epoch": 2.5525824517734907,
+      "grad_norm": 3.4119911193847656,
+      "learning_rate": 8.625461254612547e-07,
+      "loss": 1.3462,
+      "step": 2051
+    },
+    {
+      "epoch": 2.553827006845053,
+      "grad_norm": 3.5083751678466797,
+      "learning_rate": 8.602398523985241e-07,
+      "loss": 1.3632,
+      "step": 2052
+    },
+    {
+      "epoch": 2.555071561916615,
+      "grad_norm": 3.279911518096924,
+      "learning_rate": 8.579335793357934e-07,
+      "loss": 1.3326,
+      "step": 2053
+    },
+    {
+      "epoch": 2.5563161169881767,
+      "grad_norm": 3.2245192527770996,
+      "learning_rate": 8.556273062730628e-07,
+      "loss": 1.3033,
+      "step": 2054
+    },
+    {
+      "epoch": 2.5575606720597386,
+      "grad_norm": 3.3703253269195557,
+      "learning_rate": 8.533210332103322e-07,
+      "loss": 1.3219,
+      "step": 2055
+    },
+    {
+      "epoch": 2.5588052271313004,
+      "grad_norm": 3.3931314945220947,
+      "learning_rate": 8.510147601476016e-07,
+      "loss": 1.2992,
+      "step": 2056
+    },
+    {
+      "epoch": 2.5600497822028627,
+      "grad_norm": 3.5495870113372803,
+      "learning_rate": 8.487084870848709e-07,
+      "loss": 1.3872,
+      "step": 2057
+    },
+    {
+      "epoch": 2.5612943372744246,
+      "grad_norm": 3.476536273956299,
+      "learning_rate": 8.464022140221403e-07,
+      "loss": 1.3609,
+      "step": 2058
+    },
+    {
+      "epoch": 2.5625388923459864,
+      "grad_norm": 3.4503397941589355,
+      "learning_rate": 8.440959409594096e-07,
+      "loss": 1.339,
+      "step": 2059
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "grad_norm": 3.3848440647125244,
+      "learning_rate": 8.417896678966791e-07,
+      "loss": 1.3369,
+      "step": 2060
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "eval_loss": 1.61435866355896,
+      "eval_runtime": 48.4317,
+      "eval_samples_per_second": 20.648,
+      "eval_steps_per_second": 0.867,
+      "step": 2060
+    },
+    {
+      "epoch": 2.56502800248911,
+      "grad_norm": 3.474571466445923,
+      "learning_rate": 8.394833948339484e-07,
+      "loss": 1.379,
+      "step": 2061
+    },
+    {
+      "epoch": 2.566272557560672,
+      "grad_norm": 3.487356185913086,
+      "learning_rate": 8.371771217712178e-07,
+      "loss": 1.365,
+      "step": 2062
+    },
+    {
+      "epoch": 2.567517112632234,
+      "grad_norm": 3.5649149417877197,
+      "learning_rate": 8.348708487084871e-07,
+      "loss": 1.3619,
+      "step": 2063
+    },
+    {
+      "epoch": 2.5687616677037957,
+      "grad_norm": 3.572577714920044,
+      "learning_rate": 8.325645756457565e-07,
+      "loss": 1.3877,
+      "step": 2064
+    },
+    {
+      "epoch": 2.570006222775358,
+      "grad_norm": 3.3802125453948975,
+      "learning_rate": 8.302583025830259e-07,
+      "loss": 1.3323,
+      "step": 2065
+    },
+    {
+      "epoch": 2.57125077784692,
+      "grad_norm": 3.317291498184204,
+      "learning_rate": 8.279520295202953e-07,
+      "loss": 1.3532,
+      "step": 2066
+    },
+    {
+      "epoch": 2.5724953329184816,
+      "grad_norm": 3.485429525375366,
+      "learning_rate": 8.256457564575647e-07,
+      "loss": 1.3561,
+      "step": 2067
+    },
+    {
+      "epoch": 2.5737398879900435,
+      "grad_norm": 3.4251649379730225,
+      "learning_rate": 8.233394833948339e-07,
+      "loss": 1.3494,
+      "step": 2068
+    },
+    {
+      "epoch": 2.5749844430616053,
+      "grad_norm": 3.3372538089752197,
+      "learning_rate": 8.210332103321033e-07,
+      "loss": 1.3116,
+      "step": 2069
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "grad_norm": 3.3535027503967285,
+      "learning_rate": 8.187269372693728e-07,
+      "loss": 1.3541,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "eval_loss": 1.6068458557128906,
+      "eval_runtime": 51.6237,
+      "eval_samples_per_second": 19.371,
+      "eval_steps_per_second": 0.814,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5774735532047295,
+      "grad_norm": 3.4335787296295166,
+      "learning_rate": 8.164206642066422e-07,
+      "loss": 1.3884,
+      "step": 2071
+    },
+    {
+      "epoch": 2.5787181082762913,
+      "grad_norm": 3.436685800552368,
+      "learning_rate": 8.141143911439114e-07,
+      "loss": 1.3508,
+      "step": 2072
+    },
+    {
+      "epoch": 2.579962663347853,
+      "grad_norm": 3.333679676055908,
+      "learning_rate": 8.118081180811808e-07,
+      "loss": 1.3485,
+      "step": 2073
+    },
+    {
+      "epoch": 2.581207218419415,
+      "grad_norm": 3.4076309204101562,
+      "learning_rate": 8.095018450184502e-07,
+      "loss": 1.3219,
+      "step": 2074
+    },
+    {
+      "epoch": 2.582451773490977,
+      "grad_norm": 3.3851561546325684,
+      "learning_rate": 8.071955719557197e-07,
+      "loss": 1.3365,
+      "step": 2075
+    },
+    {
+      "epoch": 2.5836963285625387,
+      "grad_norm": 3.3714189529418945,
+      "learning_rate": 8.048892988929891e-07,
+      "loss": 1.3448,
+      "step": 2076
+    },
+    {
+      "epoch": 2.5849408836341006,
+      "grad_norm": 3.32585072517395,
+      "learning_rate": 8.025830258302583e-07,
+      "loss": 1.338,
+      "step": 2077
+    },
+    {
+      "epoch": 2.586185438705663,
+      "grad_norm": 3.4017553329467773,
+      "learning_rate": 8.002767527675277e-07,
+      "loss": 1.3597,
+      "step": 2078
+    },
+    {
+      "epoch": 2.5874299937772247,
+      "grad_norm": 3.334178924560547,
+      "learning_rate": 7.979704797047971e-07,
+      "loss": 1.2981,
+      "step": 2079
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "grad_norm": 3.443598985671997,
+      "learning_rate": 7.956642066420666e-07,
+      "loss": 1.3188,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "eval_loss": 1.6108036041259766,
+      "eval_runtime": 50.3752,
+      "eval_samples_per_second": 19.851,
+      "eval_steps_per_second": 0.834,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5899191039203484,
+      "grad_norm": 3.460387706756592,
+      "learning_rate": 7.933579335793358e-07,
+      "loss": 1.3794,
+      "step": 2081
+    },
+    {
+      "epoch": 2.5911636589919103,
+      "grad_norm": 3.370560646057129,
+      "learning_rate": 7.910516605166052e-07,
+      "loss": 1.3068,
+      "step": 2082
+    },
+    {
+      "epoch": 2.5924082140634725,
+      "grad_norm": 3.55534291267395,
+      "learning_rate": 7.887453874538746e-07,
+      "loss": 1.3581,
+      "step": 2083
+    },
+    {
+      "epoch": 2.5936527691350344,
+      "grad_norm": 3.322735548019409,
+      "learning_rate": 7.86439114391144e-07,
+      "loss": 1.3399,
+      "step": 2084
+    },
+    {
+      "epoch": 2.5948973242065962,
+      "grad_norm": 3.5256693363189697,
+      "learning_rate": 7.841328413284134e-07,
+      "loss": 1.377,
+      "step": 2085
+    },
+    {
+      "epoch": 2.596141879278158,
+      "grad_norm": 3.4358303546905518,
+      "learning_rate": 7.818265682656827e-07,
+      "loss": 1.3486,
+      "step": 2086
+    },
+    {
+      "epoch": 2.59738643434972,
+      "grad_norm": 3.401855230331421,
+      "learning_rate": 7.795202952029521e-07,
+      "loss": 1.3013,
+      "step": 2087
+    },
+    {
+      "epoch": 2.598630989421282,
+      "grad_norm": 3.414670467376709,
+      "learning_rate": 7.772140221402215e-07,
+      "loss": 1.3342,
+      "step": 2088
+    },
+    {
+      "epoch": 2.5998755444928436,
+      "grad_norm": 3.3348028659820557,
+      "learning_rate": 7.749077490774908e-07,
+      "loss": 1.3594,
+      "step": 2089
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "grad_norm": 3.4095754623413086,
+      "learning_rate": 7.726014760147602e-07,
+      "loss": 1.3739,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "eval_loss": 1.6106606721878052,
+      "eval_runtime": 49.3878,
+      "eval_samples_per_second": 20.248,
+      "eval_steps_per_second": 0.85,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6023646546359678,
+      "grad_norm": 3.247753143310547,
+      "learning_rate": 7.702952029520296e-07,
+      "loss": 1.3026,
+      "step": 2091
+    },
+    {
+      "epoch": 2.6036092097075296,
+      "grad_norm": 3.3973324298858643,
+      "learning_rate": 7.67988929889299e-07,
+      "loss": 1.3381,
+      "step": 2092
+    },
+    {
+      "epoch": 2.6048537647790915,
+      "grad_norm": 3.4262280464172363,
+      "learning_rate": 7.656826568265683e-07,
+      "loss": 1.3595,
+      "step": 2093
+    },
+    {
+      "epoch": 2.6060983198506533,
+      "grad_norm": 3.401784658432007,
+      "learning_rate": 7.633763837638377e-07,
+      "loss": 1.3339,
+      "step": 2094
+    },
+    {
+      "epoch": 2.607342874922215,
+      "grad_norm": 3.3231465816497803,
+      "learning_rate": 7.610701107011071e-07,
+      "loss": 1.2529,
+      "step": 2095
+    },
+    {
+      "epoch": 2.6085874299937775,
+      "grad_norm": 3.412613868713379,
+      "learning_rate": 7.587638376383765e-07,
+      "loss": 1.3341,
+      "step": 2096
+    },
+    {
+      "epoch": 2.6098319850653393,
+      "grad_norm": 3.456622362136841,
+      "learning_rate": 7.564575645756458e-07,
+      "loss": 1.3545,
+      "step": 2097
+    },
+    {
+      "epoch": 2.611076540136901,
+      "grad_norm": 3.488546133041382,
+      "learning_rate": 7.541512915129152e-07,
+      "loss": 1.3405,
+      "step": 2098
+    },
+    {
+      "epoch": 2.612321095208463,
+      "grad_norm": 3.4436495304107666,
+      "learning_rate": 7.518450184501845e-07,
+      "loss": 1.3242,
+      "step": 2099
+    },
+    {
+      "epoch": 2.613565650280025,
+      "grad_norm": 3.382150411605835,
+      "learning_rate": 7.49538745387454e-07,
+      "loss": 1.301,
+      "step": 2100
+    },
+    {
+      "epoch": 2.613565650280025,
+      "eval_loss": 1.6055735349655151,
+      "eval_runtime": 48.001,
+      "eval_samples_per_second": 20.833,
+      "eval_steps_per_second": 0.875,
+      "step": 2100
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2409,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.546349759660032e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2100/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/latest
new file mode 100644
index 0000000000000000000000000000000000000000..adbdf3f70e1e0db88a47bff3edcb0592ed0a2e61
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/latest
@@ -0,0 +1 @@
+global_step2200
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ca3af7294f2219fc064772e6af4abf0481314066
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef233d2806676c682805f85cf51776d27dc06740dc062015cc1e961a14df9e97
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9a2039c6da7a85a1210a9c30d33ef8b01f90c45a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24be9519b492ae2ab5da4e7d258ad3f276da346422161ebc6874f26057c028cd
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..c4e626fa4f7fe11f0ad555292fe2005dacbf98ad
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77a120a924d0a14713ef6ebebe2792a637dac07f7fbafa91cf0930019864eb1b
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..36ecd79206ad9e51768798cfb75723be8bbfd28d
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90ce4d29d13b8e160c3e74430a2cdac691530e61e528a4840e58dbee31cd6c91
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6
+size 17210734
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/tokenizer_config.json
@@ -0,0 +1,2086 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅂",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128258": {
+      "content": "🄿",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..54eb42636de7e1f6a30dc1d5ce649f86b37fddd4
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/trainer_state.json
@@ -0,0 +1,17193 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.7380211574362168,
+  "eval_steps": 10,
+  "global_step": 2200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8368,
+      "step": 1
+    },
+    {
+      "epoch": 0.002489110143123833,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.86,
+      "step": 2
+    },
+    {
+      "epoch": 0.00373366521468575,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8265,
+      "step": 3
+    },
+    {
+      "epoch": 0.004978220286247666,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8271,
+      "step": 4
+    },
+    {
+      "epoch": 0.006222775357809583,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8768,
+      "step": 5
+    },
+    {
+      "epoch": 0.0074673304293715,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8402,
+      "step": 6
+    },
+    {
+      "epoch": 0.008711885500933417,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.785,
+      "step": 7
+    },
+    {
+      "epoch": 0.009956440572495333,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8466,
+      "step": 8
+    },
+    {
+      "epoch": 0.01120099564405725,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.7673,
+      "step": 9
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "eval_loss": 2.8535053730010986,
+      "eval_runtime": 42.446,
+      "eval_samples_per_second": 23.559,
+      "eval_steps_per_second": 0.989,
+      "step": 10
+    },
+    {
+      "epoch": 0.013690105787181083,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8592,
+      "step": 11
+    },
+    {
+      "epoch": 0.014934660858743,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.804,
+      "step": 12
+    },
+    {
+      "epoch": 0.016179215930304917,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8076,
+      "step": 13
+    },
+    {
+      "epoch": 0.017423771001866834,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8219,
+      "step": 14
+    },
+    {
+      "epoch": 0.018668326073428748,
+      "grad_norm": 5.105710506439209,
+      "learning_rate": 2.0746887966804982e-08,
+      "loss": 2.7995,
+      "step": 15
+    },
+    {
+      "epoch": 0.019912881144990666,
+      "grad_norm": 5.156588077545166,
+      "learning_rate": 4.1493775933609963e-08,
+      "loss": 2.8577,
+      "step": 16
+    },
+    {
+      "epoch": 0.021157436216552583,
+      "grad_norm": 5.709373950958252,
+      "learning_rate": 6.224066390041494e-08,
+      "loss": 2.8433,
+      "step": 17
+    },
+    {
+      "epoch": 0.0224019912881145,
+      "grad_norm": 5.64658784866333,
+      "learning_rate": 8.298755186721993e-08,
+      "loss": 2.8428,
+      "step": 18
+    },
+    {
+      "epoch": 0.023646546359676415,
+      "grad_norm": 5.541937351226807,
+      "learning_rate": 1.037344398340249e-07,
+      "loss": 2.7785,
+      "step": 19
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "grad_norm": 5.233374118804932,
+      "learning_rate": 1.2448132780082988e-07,
+      "loss": 2.8599,
+      "step": 20
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "eval_loss": 2.8521182537078857,
+      "eval_runtime": 43.5664,
+      "eval_samples_per_second": 22.953,
+      "eval_steps_per_second": 0.964,
+      "step": 20
+    },
+    {
+      "epoch": 0.02613565650280025,
+      "grad_norm": 5.407598972320557,
+      "learning_rate": 1.4522821576763488e-07,
+      "loss": 2.8081,
+      "step": 21
+    },
+    {
+      "epoch": 0.027380211574362167,
+      "grad_norm": 5.0483317375183105,
+      "learning_rate": 1.6597510373443985e-07,
+      "loss": 2.8489,
+      "step": 22
+    },
+    {
+      "epoch": 0.02862476664592408,
+      "grad_norm": 4.985588550567627,
+      "learning_rate": 1.8672199170124483e-07,
+      "loss": 2.8375,
+      "step": 23
+    },
+    {
+      "epoch": 0.029869321717486,
+      "grad_norm": 5.201700687408447,
+      "learning_rate": 2.074688796680498e-07,
+      "loss": 2.8355,
+      "step": 24
+    },
+    {
+      "epoch": 0.031113876789047916,
+      "grad_norm": 5.009235382080078,
+      "learning_rate": 2.2821576763485478e-07,
+      "loss": 2.7774,
+      "step": 25
+    },
+    {
+      "epoch": 0.03235843186060983,
+      "grad_norm": 4.509210109710693,
+      "learning_rate": 2.4896265560165975e-07,
+      "loss": 2.7992,
+      "step": 26
+    },
+    {
+      "epoch": 0.03360298693217175,
+      "grad_norm": 4.529262065887451,
+      "learning_rate": 2.6970954356846476e-07,
+      "loss": 2.836,
+      "step": 27
+    },
+    {
+      "epoch": 0.03484754200373367,
+      "grad_norm": 4.526787281036377,
+      "learning_rate": 2.9045643153526976e-07,
+      "loss": 2.8128,
+      "step": 28
+    },
+    {
+      "epoch": 0.03609209707529558,
+      "grad_norm": 4.016609191894531,
+      "learning_rate": 3.112033195020747e-07,
+      "loss": 2.7947,
+      "step": 29
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "grad_norm": 3.4784562587738037,
+      "learning_rate": 3.319502074688797e-07,
+      "loss": 2.7854,
+      "step": 30
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "eval_loss": 2.8020100593566895,
+      "eval_runtime": 44.8464,
+      "eval_samples_per_second": 22.298,
+      "eval_steps_per_second": 0.937,
+      "step": 30
+    },
+    {
+      "epoch": 0.038581207218419414,
+      "grad_norm": 3.4887945652008057,
+      "learning_rate": 3.5269709543568466e-07,
+      "loss": 2.7908,
+      "step": 31
+    },
+    {
+      "epoch": 0.03982576228998133,
+      "grad_norm": 3.4533374309539795,
+      "learning_rate": 3.7344398340248966e-07,
+      "loss": 2.7358,
+      "step": 32
+    },
+    {
+      "epoch": 0.04107031736154325,
+      "grad_norm": 3.270954132080078,
+      "learning_rate": 3.941908713692946e-07,
+      "loss": 2.8013,
+      "step": 33
+    },
+    {
+      "epoch": 0.042314872433105166,
+      "grad_norm": 3.61322283744812,
+      "learning_rate": 4.149377593360996e-07,
+      "loss": 2.7521,
+      "step": 34
+    },
+    {
+      "epoch": 0.043559427504667084,
+      "grad_norm": 3.158719062805176,
+      "learning_rate": 4.3568464730290456e-07,
+      "loss": 2.7573,
+      "step": 35
+    },
+    {
+      "epoch": 0.044803982576229,
+      "grad_norm": 2.888746500015259,
+      "learning_rate": 4.5643153526970956e-07,
+      "loss": 2.7795,
+      "step": 36
+    },
+    {
+      "epoch": 0.04604853764779091,
+      "grad_norm": 3.022629499435425,
+      "learning_rate": 4.771784232365145e-07,
+      "loss": 2.7891,
+      "step": 37
+    },
+    {
+      "epoch": 0.04729309271935283,
+      "grad_norm": 3.038097381591797,
+      "learning_rate": 4.979253112033195e-07,
+      "loss": 2.7376,
+      "step": 38
+    },
+    {
+      "epoch": 0.04853764779091475,
+      "grad_norm": 2.9992239475250244,
+      "learning_rate": 5.186721991701245e-07,
+      "loss": 2.7279,
+      "step": 39
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "grad_norm": 2.908842086791992,
+      "learning_rate": 5.394190871369295e-07,
+      "loss": 2.6582,
+      "step": 40
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "eval_loss": 2.725968837738037,
+      "eval_runtime": 44.9498,
+      "eval_samples_per_second": 22.247,
+      "eval_steps_per_second": 0.934,
+      "step": 40
+    },
+    {
+      "epoch": 0.05102675793403858,
+      "grad_norm": 2.8025174140930176,
+      "learning_rate": 5.601659751037345e-07,
+      "loss": 2.6809,
+      "step": 41
+    },
+    {
+      "epoch": 0.0522713130056005,
+      "grad_norm": 2.7555699348449707,
+      "learning_rate": 5.809128630705395e-07,
+      "loss": 2.6954,
+      "step": 42
+    },
+    {
+      "epoch": 0.053515868077162417,
+      "grad_norm": 2.4732933044433594,
+      "learning_rate": 6.016597510373444e-07,
+      "loss": 2.7307,
+      "step": 43
+    },
+    {
+      "epoch": 0.054760423148724334,
+      "grad_norm": 2.451366424560547,
+      "learning_rate": 6.224066390041494e-07,
+      "loss": 2.7064,
+      "step": 44
+    },
+    {
+      "epoch": 0.056004978220286245,
+      "grad_norm": 2.5768678188323975,
+      "learning_rate": 6.431535269709543e-07,
+      "loss": 2.6466,
+      "step": 45
+    },
+    {
+      "epoch": 0.05724953329184816,
+      "grad_norm": 2.579332113265991,
+      "learning_rate": 6.639004149377594e-07,
+      "loss": 2.6127,
+      "step": 46
+    },
+    {
+      "epoch": 0.05849408836341008,
+      "grad_norm": 2.231207847595215,
+      "learning_rate": 6.846473029045644e-07,
+      "loss": 2.6429,
+      "step": 47
+    },
+    {
+      "epoch": 0.059738643434972,
+      "grad_norm": 2.134375810623169,
+      "learning_rate": 7.053941908713693e-07,
+      "loss": 2.6423,
+      "step": 48
+    },
+    {
+      "epoch": 0.060983198506533914,
+      "grad_norm": 2.0485494136810303,
+      "learning_rate": 7.261410788381744e-07,
+      "loss": 2.581,
+      "step": 49
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "grad_norm": 2.02909517288208,
+      "learning_rate": 7.468879668049793e-07,
+      "loss": 2.6357,
+      "step": 50
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "eval_loss": 2.632822036743164,
+      "eval_runtime": 47.0104,
+      "eval_samples_per_second": 21.272,
+      "eval_steps_per_second": 0.893,
+      "step": 50
+    },
+    {
+      "epoch": 0.06347230864965775,
+      "grad_norm": 1.892500877380371,
+      "learning_rate": 7.676348547717843e-07,
+      "loss": 2.5945,
+      "step": 51
+    },
+    {
+      "epoch": 0.06471686372121967,
+      "grad_norm": 2.107921600341797,
+      "learning_rate": 7.883817427385892e-07,
+      "loss": 2.5867,
+      "step": 52
+    },
+    {
+      "epoch": 0.06596141879278158,
+      "grad_norm": 1.8287527561187744,
+      "learning_rate": 8.091286307053943e-07,
+      "loss": 2.5862,
+      "step": 53
+    },
+    {
+      "epoch": 0.0672059738643435,
+      "grad_norm": 1.7342065572738647,
+      "learning_rate": 8.298755186721992e-07,
+      "loss": 2.5735,
+      "step": 54
+    },
+    {
+      "epoch": 0.06845052893590542,
+      "grad_norm": 1.7905038595199585,
+      "learning_rate": 8.506224066390042e-07,
+      "loss": 2.572,
+      "step": 55
+    },
+    {
+      "epoch": 0.06969508400746734,
+      "grad_norm": 1.9715144634246826,
+      "learning_rate": 8.713692946058091e-07,
+      "loss": 2.5797,
+      "step": 56
+    },
+    {
+      "epoch": 0.07093963907902924,
+      "grad_norm": 1.788966417312622,
+      "learning_rate": 8.921161825726142e-07,
+      "loss": 2.5567,
+      "step": 57
+    },
+    {
+      "epoch": 0.07218419415059116,
+      "grad_norm": 1.8978915214538574,
+      "learning_rate": 9.128630705394191e-07,
+      "loss": 2.5776,
+      "step": 58
+    },
+    {
+      "epoch": 0.07342874922215308,
+      "grad_norm": 1.6401689052581787,
+      "learning_rate": 9.336099585062241e-07,
+      "loss": 2.5662,
+      "step": 59
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "grad_norm": 1.8200798034667969,
+      "learning_rate": 9.54356846473029e-07,
+      "loss": 2.5001,
+      "step": 60
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "eval_loss": 2.569110870361328,
+      "eval_runtime": 47.3292,
+      "eval_samples_per_second": 21.129,
+      "eval_steps_per_second": 0.887,
+      "step": 60
+    },
+    {
+      "epoch": 0.07591785936527691,
+      "grad_norm": 1.5468674898147583,
+      "learning_rate": 9.751037344398341e-07,
+      "loss": 2.5308,
+      "step": 61
+    },
+    {
+      "epoch": 0.07716241443683883,
+      "grad_norm": 1.723684310913086,
+      "learning_rate": 9.95850622406639e-07,
+      "loss": 2.4907,
+      "step": 62
+    },
+    {
+      "epoch": 0.07840696950840075,
+      "grad_norm": 1.6346293687820435,
+      "learning_rate": 1.0165975103734441e-06,
+      "loss": 2.514,
+      "step": 63
+    },
+    {
+      "epoch": 0.07965152457996266,
+      "grad_norm": 1.8902088403701782,
+      "learning_rate": 1.037344398340249e-06,
+      "loss": 2.4859,
+      "step": 64
+    },
+    {
+      "epoch": 0.08089607965152458,
+      "grad_norm": 2.092611312866211,
+      "learning_rate": 1.058091286307054e-06,
+      "loss": 2.4888,
+      "step": 65
+    },
+    {
+      "epoch": 0.0821406347230865,
+      "grad_norm": 1.758482575416565,
+      "learning_rate": 1.078838174273859e-06,
+      "loss": 2.4537,
+      "step": 66
+    },
+    {
+      "epoch": 0.08338518979464841,
+      "grad_norm": 1.8941714763641357,
+      "learning_rate": 1.099585062240664e-06,
+      "loss": 2.5416,
+      "step": 67
+    },
+    {
+      "epoch": 0.08462974486621033,
+      "grad_norm": 1.7030234336853027,
+      "learning_rate": 1.120331950207469e-06,
+      "loss": 2.4791,
+      "step": 68
+    },
+    {
+      "epoch": 0.08587429993777225,
+      "grad_norm": 1.7602269649505615,
+      "learning_rate": 1.141078838174274e-06,
+      "loss": 2.4316,
+      "step": 69
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "grad_norm": 1.7635431289672852,
+      "learning_rate": 1.161825726141079e-06,
+      "loss": 2.4674,
+      "step": 70
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "eval_loss": 2.4922080039978027,
+      "eval_runtime": 45.3028,
+      "eval_samples_per_second": 22.074,
+      "eval_steps_per_second": 0.927,
+      "step": 70
+    },
+    {
+      "epoch": 0.08836341008089608,
+      "grad_norm": 2.0913803577423096,
+      "learning_rate": 1.182572614107884e-06,
+      "loss": 2.4341,
+      "step": 71
+    },
+    {
+      "epoch": 0.089607965152458,
+      "grad_norm": 1.8292183876037598,
+      "learning_rate": 1.2033195020746888e-06,
+      "loss": 2.4291,
+      "step": 72
+    },
+    {
+      "epoch": 0.09085252022401992,
+      "grad_norm": 2.206770181655884,
+      "learning_rate": 1.224066390041494e-06,
+      "loss": 2.3764,
+      "step": 73
+    },
+    {
+      "epoch": 0.09209707529558182,
+      "grad_norm": 1.8263559341430664,
+      "learning_rate": 1.2448132780082988e-06,
+      "loss": 2.4082,
+      "step": 74
+    },
+    {
+      "epoch": 0.09334163036714374,
+      "grad_norm": 1.9405455589294434,
+      "learning_rate": 1.2655601659751037e-06,
+      "loss": 2.4673,
+      "step": 75
+    },
+    {
+      "epoch": 0.09458618543870566,
+      "grad_norm": 2.138108015060425,
+      "learning_rate": 1.2863070539419086e-06,
+      "loss": 2.3879,
+      "step": 76
+    },
+    {
+      "epoch": 0.09583074051026758,
+      "grad_norm": 1.9024745225906372,
+      "learning_rate": 1.307053941908714e-06,
+      "loss": 2.4344,
+      "step": 77
+    },
+    {
+      "epoch": 0.0970752955818295,
+      "grad_norm": 1.9136689901351929,
+      "learning_rate": 1.3278008298755188e-06,
+      "loss": 2.474,
+      "step": 78
+    },
+    {
+      "epoch": 0.09831985065339141,
+      "grad_norm": 2.4752795696258545,
+      "learning_rate": 1.3485477178423237e-06,
+      "loss": 2.3412,
+      "step": 79
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "grad_norm": 1.8720006942749023,
+      "learning_rate": 1.3692946058091288e-06,
+      "loss": 2.3438,
+      "step": 80
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "eval_loss": 2.3980512619018555,
+      "eval_runtime": 48.2788,
+      "eval_samples_per_second": 20.713,
+      "eval_steps_per_second": 0.87,
+      "step": 80
+    },
+    {
+      "epoch": 0.10080896079651525,
+      "grad_norm": 2.671691656112671,
+      "learning_rate": 1.3900414937759337e-06,
+      "loss": 2.3336,
+      "step": 81
+    },
+    {
+      "epoch": 0.10205351586807716,
+      "grad_norm": 2.2953391075134277,
+      "learning_rate": 1.4107883817427386e-06,
+      "loss": 2.377,
+      "step": 82
+    },
+    {
+      "epoch": 0.10329807093963908,
+      "grad_norm": 3.009018898010254,
+      "learning_rate": 1.4315352697095435e-06,
+      "loss": 2.2977,
+      "step": 83
+    },
+    {
+      "epoch": 0.104542626011201,
+      "grad_norm": 2.664454936981201,
+      "learning_rate": 1.4522821576763488e-06,
+      "loss": 2.3271,
+      "step": 84
+    },
+    {
+      "epoch": 0.10578718108276292,
+      "grad_norm": 3.017303705215454,
+      "learning_rate": 1.4730290456431537e-06,
+      "loss": 2.3251,
+      "step": 85
+    },
+    {
+      "epoch": 0.10703173615432483,
+      "grad_norm": 2.634716510772705,
+      "learning_rate": 1.4937759336099586e-06,
+      "loss": 2.332,
+      "step": 86
+    },
+    {
+      "epoch": 0.10827629122588675,
+      "grad_norm": 3.059644937515259,
+      "learning_rate": 1.5145228215767635e-06,
+      "loss": 2.3478,
+      "step": 87
+    },
+    {
+      "epoch": 0.10952084629744867,
+      "grad_norm": 2.6962637901306152,
+      "learning_rate": 1.5352697095435686e-06,
+      "loss": 2.2792,
+      "step": 88
+    },
+    {
+      "epoch": 0.11076540136901059,
+      "grad_norm": 3.419729709625244,
+      "learning_rate": 1.5560165975103735e-06,
+      "loss": 2.2571,
+      "step": 89
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "grad_norm": 2.740781545639038,
+      "learning_rate": 1.5767634854771784e-06,
+      "loss": 2.2875,
+      "step": 90
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "eval_loss": 2.30843186378479,
+      "eval_runtime": 49.9264,
+      "eval_samples_per_second": 20.029,
+      "eval_steps_per_second": 0.841,
+      "step": 90
+    },
+    {
+      "epoch": 0.1132545115121344,
+      "grad_norm": 2.5608789920806885,
+      "learning_rate": 1.5975103734439833e-06,
+      "loss": 2.2592,
+      "step": 91
+    },
+    {
+      "epoch": 0.11449906658369632,
+      "grad_norm": 2.687999963760376,
+      "learning_rate": 1.6182572614107886e-06,
+      "loss": 2.2546,
+      "step": 92
+    },
+    {
+      "epoch": 0.11574362165525824,
+      "grad_norm": 2.695909023284912,
+      "learning_rate": 1.6390041493775935e-06,
+      "loss": 2.2525,
+      "step": 93
+    },
+    {
+      "epoch": 0.11698817672682016,
+      "grad_norm": 2.818357467651367,
+      "learning_rate": 1.6597510373443984e-06,
+      "loss": 2.216,
+      "step": 94
+    },
+    {
+      "epoch": 0.11823273179838208,
+      "grad_norm": 2.884119987487793,
+      "learning_rate": 1.6804979253112035e-06,
+      "loss": 2.2321,
+      "step": 95
+    },
+    {
+      "epoch": 0.119477286869944,
+      "grad_norm": 2.52104115486145,
+      "learning_rate": 1.7012448132780084e-06,
+      "loss": 2.199,
+      "step": 96
+    },
+    {
+      "epoch": 0.12072184194150591,
+      "grad_norm": 2.420313596725464,
+      "learning_rate": 1.7219917012448133e-06,
+      "loss": 2.1862,
+      "step": 97
+    },
+    {
+      "epoch": 0.12196639701306783,
+      "grad_norm": 2.8047542572021484,
+      "learning_rate": 1.7427385892116182e-06,
+      "loss": 2.1793,
+      "step": 98
+    },
+    {
+      "epoch": 0.12321095208462975,
+      "grad_norm": 2.836482286453247,
+      "learning_rate": 1.7634854771784235e-06,
+      "loss": 2.2271,
+      "step": 99
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "grad_norm": 2.5282301902770996,
+      "learning_rate": 1.7842323651452284e-06,
+      "loss": 2.1768,
+      "step": 100
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "eval_loss": 2.2312686443328857,
+      "eval_runtime": 49.2382,
+      "eval_samples_per_second": 20.309,
+      "eval_steps_per_second": 0.853,
+      "step": 100
+    },
+    {
+      "epoch": 0.12570006222775357,
+      "grad_norm": 3.3407280445098877,
+      "learning_rate": 1.8049792531120333e-06,
+      "loss": 2.1666,
+      "step": 101
+    },
+    {
+      "epoch": 0.1269446172993155,
+      "grad_norm": 2.4754133224487305,
+      "learning_rate": 1.8257261410788382e-06,
+      "loss": 2.1768,
+      "step": 102
+    },
+    {
+      "epoch": 0.1281891723708774,
+      "grad_norm": 3.430889129638672,
+      "learning_rate": 1.8464730290456433e-06,
+      "loss": 2.1953,
+      "step": 103
+    },
+    {
+      "epoch": 0.12943372744243933,
+      "grad_norm": 2.835294246673584,
+      "learning_rate": 1.8672199170124482e-06,
+      "loss": 2.146,
+      "step": 104
+    },
+    {
+      "epoch": 0.13067828251400124,
+      "grad_norm": 3.1532323360443115,
+      "learning_rate": 1.8879668049792531e-06,
+      "loss": 2.1729,
+      "step": 105
+    },
+    {
+      "epoch": 0.13192283758556317,
+      "grad_norm": 3.2278342247009277,
+      "learning_rate": 1.908713692946058e-06,
+      "loss": 2.1336,
+      "step": 106
+    },
+    {
+      "epoch": 0.13316739265712507,
+      "grad_norm": 2.7892515659332275,
+      "learning_rate": 1.929460580912863e-06,
+      "loss": 2.141,
+      "step": 107
+    },
+    {
+      "epoch": 0.134411947728687,
+      "grad_norm": 3.0179977416992188,
+      "learning_rate": 1.9502074688796682e-06,
+      "loss": 2.1255,
+      "step": 108
+    },
+    {
+      "epoch": 0.1356565028002489,
+      "grad_norm": 2.977935552597046,
+      "learning_rate": 1.970954356846473e-06,
+      "loss": 2.0986,
+      "step": 109
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "grad_norm": 3.3042492866516113,
+      "learning_rate": 1.991701244813278e-06,
+      "loss": 2.0992,
+      "step": 110
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "eval_loss": 2.171299695968628,
+      "eval_runtime": 42.9511,
+      "eval_samples_per_second": 23.282,
+      "eval_steps_per_second": 0.978,
+      "step": 110
+    },
+    {
+      "epoch": 0.13814561294337274,
+      "grad_norm": 2.9750890731811523,
+      "learning_rate": 2.012448132780083e-06,
+      "loss": 2.1116,
+      "step": 111
+    },
+    {
+      "epoch": 0.13939016801493467,
+      "grad_norm": 2.8199286460876465,
+      "learning_rate": 2.0331950207468883e-06,
+      "loss": 2.0822,
+      "step": 112
+    },
+    {
+      "epoch": 0.14063472308649658,
+      "grad_norm": 2.930532217025757,
+      "learning_rate": 2.053941908713693e-06,
+      "loss": 2.161,
+      "step": 113
+    },
+    {
+      "epoch": 0.14187927815805848,
+      "grad_norm": 3.3400321006774902,
+      "learning_rate": 2.074688796680498e-06,
+      "loss": 2.0953,
+      "step": 114
+    },
+    {
+      "epoch": 0.1431238332296204,
+      "grad_norm": 2.716564655303955,
+      "learning_rate": 2.095435684647303e-06,
+      "loss": 2.0757,
+      "step": 115
+    },
+    {
+      "epoch": 0.14436838830118232,
+      "grad_norm": 2.7586236000061035,
+      "learning_rate": 2.116182572614108e-06,
+      "loss": 2.0979,
+      "step": 116
+    },
+    {
+      "epoch": 0.14561294337274425,
+      "grad_norm": 2.622126340866089,
+      "learning_rate": 2.136929460580913e-06,
+      "loss": 2.0748,
+      "step": 117
+    },
+    {
+      "epoch": 0.14685749844430615,
+      "grad_norm": 2.646477460861206,
+      "learning_rate": 2.157676348547718e-06,
+      "loss": 2.0605,
+      "step": 118
+    },
+    {
+      "epoch": 0.14810205351586808,
+      "grad_norm": 2.8882410526275635,
+      "learning_rate": 2.178423236514523e-06,
+      "loss": 2.0501,
+      "step": 119
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "grad_norm": 2.8302247524261475,
+      "learning_rate": 2.199170124481328e-06,
+      "loss": 2.0592,
+      "step": 120
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "eval_loss": 2.1291966438293457,
+      "eval_runtime": 47.5399,
+      "eval_samples_per_second": 21.035,
+      "eval_steps_per_second": 0.883,
+      "step": 120
+    },
+    {
+      "epoch": 0.15059116365899192,
+      "grad_norm": 3.26979660987854,
+      "learning_rate": 2.219917012448133e-06,
+      "loss": 2.0422,
+      "step": 121
+    },
+    {
+      "epoch": 0.15183571873055382,
+      "grad_norm": 2.7483913898468018,
+      "learning_rate": 2.240663900414938e-06,
+      "loss": 2.0872,
+      "step": 122
+    },
+    {
+      "epoch": 0.15308027380211575,
+      "grad_norm": 3.1252024173736572,
+      "learning_rate": 2.2614107883817427e-06,
+      "loss": 2.0684,
+      "step": 123
+    },
+    {
+      "epoch": 0.15432482887367766,
+      "grad_norm": 2.9175283908843994,
+      "learning_rate": 2.282157676348548e-06,
+      "loss": 2.0522,
+      "step": 124
+    },
+    {
+      "epoch": 0.1555693839452396,
+      "grad_norm": 2.7950754165649414,
+      "learning_rate": 2.302904564315353e-06,
+      "loss": 2.065,
+      "step": 125
+    },
+    {
+      "epoch": 0.1568139390168015,
+      "grad_norm": 2.84049654006958,
+      "learning_rate": 2.323651452282158e-06,
+      "loss": 2.0781,
+      "step": 126
+    },
+    {
+      "epoch": 0.15805849408836342,
+      "grad_norm": 2.7396647930145264,
+      "learning_rate": 2.3443983402489627e-06,
+      "loss": 2.1758,
+      "step": 127
+    },
+    {
+      "epoch": 0.15930304915992533,
+      "grad_norm": 3.0765926837921143,
+      "learning_rate": 2.365145228215768e-06,
+      "loss": 2.0172,
+      "step": 128
+    },
+    {
+      "epoch": 0.16054760423148726,
+      "grad_norm": 3.217189073562622,
+      "learning_rate": 2.385892116182573e-06,
+      "loss": 2.0286,
+      "step": 129
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "grad_norm": 3.141545295715332,
+      "learning_rate": 2.4066390041493776e-06,
+      "loss": 2.0583,
+      "step": 130
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "eval_loss": 2.0946149826049805,
+      "eval_runtime": 54.92,
+      "eval_samples_per_second": 18.208,
+      "eval_steps_per_second": 0.765,
+      "step": 130
+    },
+    {
+      "epoch": 0.16303671437461106,
+      "grad_norm": 2.850052833557129,
+      "learning_rate": 2.4273858921161828e-06,
+      "loss": 2.0746,
+      "step": 131
+    },
+    {
+      "epoch": 0.164281269446173,
+      "grad_norm": 3.28913950920105,
+      "learning_rate": 2.448132780082988e-06,
+      "loss": 2.0539,
+      "step": 132
+    },
+    {
+      "epoch": 0.1655258245177349,
+      "grad_norm": 2.8819124698638916,
+      "learning_rate": 2.468879668049793e-06,
+      "loss": 1.9843,
+      "step": 133
+    },
+    {
+      "epoch": 0.16677037958929683,
+      "grad_norm": 3.6254632472991943,
+      "learning_rate": 2.4896265560165977e-06,
+      "loss": 2.0233,
+      "step": 134
+    },
+    {
+      "epoch": 0.16801493466085873,
+      "grad_norm": 2.7385146617889404,
+      "learning_rate": 2.5103734439834028e-06,
+      "loss": 1.9667,
+      "step": 135
+    },
+    {
+      "epoch": 0.16925948973242066,
+      "grad_norm": 2.9722647666931152,
+      "learning_rate": 2.5311203319502074e-06,
+      "loss": 1.9527,
+      "step": 136
+    },
+    {
+      "epoch": 0.17050404480398257,
+      "grad_norm": 3.139526605606079,
+      "learning_rate": 2.5518672199170125e-06,
+      "loss": 1.9967,
+      "step": 137
+    },
+    {
+      "epoch": 0.1717485998755445,
+      "grad_norm": 3.399920701980591,
+      "learning_rate": 2.5726141078838172e-06,
+      "loss": 1.9798,
+      "step": 138
+    },
+    {
+      "epoch": 0.1729931549471064,
+      "grad_norm": 2.8017327785491943,
+      "learning_rate": 2.5933609958506228e-06,
+      "loss": 1.983,
+      "step": 139
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "grad_norm": 3.9967198371887207,
+      "learning_rate": 2.614107883817428e-06,
+      "loss": 1.977,
+      "step": 140
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "eval_loss": 2.059037685394287,
+      "eval_runtime": 52.6232,
+      "eval_samples_per_second": 19.003,
+      "eval_steps_per_second": 0.798,
+      "step": 140
+    },
+    {
+      "epoch": 0.17548226509023024,
+      "grad_norm": 2.893092393875122,
+      "learning_rate": 2.6348547717842326e-06,
+      "loss": 1.9888,
+      "step": 141
+    },
+    {
+      "epoch": 0.17672682016179217,
+      "grad_norm": 3.634352207183838,
+      "learning_rate": 2.6556016597510377e-06,
+      "loss": 2.0726,
+      "step": 142
+    },
+    {
+      "epoch": 0.17797137523335407,
+      "grad_norm": 3.3651444911956787,
+      "learning_rate": 2.6763485477178423e-06,
+      "loss": 1.973,
+      "step": 143
+    },
+    {
+      "epoch": 0.179215930304916,
+      "grad_norm": 3.968986988067627,
+      "learning_rate": 2.6970954356846475e-06,
+      "loss": 1.9948,
+      "step": 144
+    },
+    {
+      "epoch": 0.1804604853764779,
+      "grad_norm": 3.20105242729187,
+      "learning_rate": 2.717842323651452e-06,
+      "loss": 1.9238,
+      "step": 145
+    },
+    {
+      "epoch": 0.18170504044803984,
+      "grad_norm": 3.648339033126831,
+      "learning_rate": 2.7385892116182577e-06,
+      "loss": 1.9923,
+      "step": 146
+    },
+    {
+      "epoch": 0.18294959551960174,
+      "grad_norm": 3.0856316089630127,
+      "learning_rate": 2.7593360995850628e-06,
+      "loss": 1.9749,
+      "step": 147
+    },
+    {
+      "epoch": 0.18419415059116365,
+      "grad_norm": 3.2681071758270264,
+      "learning_rate": 2.7800829875518675e-06,
+      "loss": 1.9304,
+      "step": 148
+    },
+    {
+      "epoch": 0.18543870566272558,
+      "grad_norm": 2.634958267211914,
+      "learning_rate": 2.8008298755186726e-06,
+      "loss": 1.9237,
+      "step": 149
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "grad_norm": 2.769491672515869,
+      "learning_rate": 2.8215767634854773e-06,
+      "loss": 1.8963,
+      "step": 150
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "eval_loss": 2.0394654273986816,
+      "eval_runtime": 55.2136,
+      "eval_samples_per_second": 18.111,
+      "eval_steps_per_second": 0.761,
+      "step": 150
+    },
+    {
+      "epoch": 0.1879278158058494,
+      "grad_norm": 3.4345781803131104,
+      "learning_rate": 2.8423236514522824e-06,
+      "loss": 1.9836,
+      "step": 151
+    },
+    {
+      "epoch": 0.18917237087741132,
+      "grad_norm": 3.115727424621582,
+      "learning_rate": 2.863070539419087e-06,
+      "loss": 1.9352,
+      "step": 152
+    },
+    {
+      "epoch": 0.19041692594897325,
+      "grad_norm": 3.050652503967285,
+      "learning_rate": 2.883817427385892e-06,
+      "loss": 1.9031,
+      "step": 153
+    },
+    {
+      "epoch": 0.19166148102053515,
+      "grad_norm": 2.9404428005218506,
+      "learning_rate": 2.9045643153526977e-06,
+      "loss": 1.9196,
+      "step": 154
+    },
+    {
+      "epoch": 0.19290603609209708,
+      "grad_norm": 3.080810785293579,
+      "learning_rate": 2.9253112033195024e-06,
+      "loss": 1.9405,
+      "step": 155
+    },
+    {
+      "epoch": 0.194150591163659,
+      "grad_norm": 3.058558702468872,
+      "learning_rate": 2.9460580912863075e-06,
+      "loss": 1.9052,
+      "step": 156
+    },
+    {
+      "epoch": 0.19539514623522092,
+      "grad_norm": 3.307955026626587,
+      "learning_rate": 2.966804979253112e-06,
+      "loss": 1.9245,
+      "step": 157
+    },
+    {
+      "epoch": 0.19663970130678282,
+      "grad_norm": 2.845506191253662,
+      "learning_rate": 2.9875518672199173e-06,
+      "loss": 1.9461,
+      "step": 158
+    },
+    {
+      "epoch": 0.19788425637834475,
+      "grad_norm": 2.758654832839966,
+      "learning_rate": 3.008298755186722e-06,
+      "loss": 1.8885,
+      "step": 159
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "grad_norm": 3.161252737045288,
+      "learning_rate": 3.029045643153527e-06,
+      "loss": 1.8535,
+      "step": 160
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "eval_loss": 2.018101453781128,
+      "eval_runtime": 51.0744,
+      "eval_samples_per_second": 19.579,
+      "eval_steps_per_second": 0.822,
+      "step": 160
+    },
+    {
+      "epoch": 0.2003733665214686,
+      "grad_norm": 3.196847677230835,
+      "learning_rate": 3.0497925311203326e-06,
+      "loss": 1.8853,
+      "step": 161
+    },
+    {
+      "epoch": 0.2016179215930305,
+      "grad_norm": 2.8364663124084473,
+      "learning_rate": 3.0705394190871373e-06,
+      "loss": 1.9463,
+      "step": 162
+    },
+    {
+      "epoch": 0.2028624766645924,
+      "grad_norm": 3.0874054431915283,
+      "learning_rate": 3.0912863070539424e-06,
+      "loss": 1.9254,
+      "step": 163
+    },
+    {
+      "epoch": 0.20410703173615433,
+      "grad_norm": 2.7914493083953857,
+      "learning_rate": 3.112033195020747e-06,
+      "loss": 1.9213,
+      "step": 164
+    },
+    {
+      "epoch": 0.20535158680771623,
+      "grad_norm": 3.3871428966522217,
+      "learning_rate": 3.132780082987552e-06,
+      "loss": 1.8993,
+      "step": 165
+    },
+    {
+      "epoch": 0.20659614187927816,
+      "grad_norm": 3.096653461456299,
+      "learning_rate": 3.153526970954357e-06,
+      "loss": 1.8827,
+      "step": 166
+    },
+    {
+      "epoch": 0.20784069695084006,
+      "grad_norm": 2.8276076316833496,
+      "learning_rate": 3.174273858921162e-06,
+      "loss": 1.912,
+      "step": 167
+    },
+    {
+      "epoch": 0.209085252022402,
+      "grad_norm": 3.3058435916900635,
+      "learning_rate": 3.1950207468879666e-06,
+      "loss": 1.8381,
+      "step": 168
+    },
+    {
+      "epoch": 0.2103298070939639,
+      "grad_norm": 3.5017333030700684,
+      "learning_rate": 3.215767634854772e-06,
+      "loss": 1.9544,
+      "step": 169
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "grad_norm": 3.457296133041382,
+      "learning_rate": 3.2365145228215773e-06,
+      "loss": 1.922,
+      "step": 170
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "eval_loss": 1.988455891609192,
+      "eval_runtime": 56.2099,
+      "eval_samples_per_second": 17.79,
+      "eval_steps_per_second": 0.747,
+      "step": 170
+    },
+    {
+      "epoch": 0.21281891723708773,
+      "grad_norm": 3.1964566707611084,
+      "learning_rate": 3.257261410788382e-06,
+      "loss": 1.8925,
+      "step": 171
+    },
+    {
+      "epoch": 0.21406347230864967,
+      "grad_norm": 3.234652042388916,
+      "learning_rate": 3.278008298755187e-06,
+      "loss": 1.884,
+      "step": 172
+    },
+    {
+      "epoch": 0.21530802738021157,
+      "grad_norm": 3.14414119720459,
+      "learning_rate": 3.2987551867219918e-06,
+      "loss": 1.8905,
+      "step": 173
+    },
+    {
+      "epoch": 0.2165525824517735,
+      "grad_norm": 3.606379508972168,
+      "learning_rate": 3.319502074688797e-06,
+      "loss": 1.946,
+      "step": 174
+    },
+    {
+      "epoch": 0.2177971375233354,
+      "grad_norm": 2.986646890640259,
+      "learning_rate": 3.3402489626556016e-06,
+      "loss": 1.8691,
+      "step": 175
+    },
+    {
+      "epoch": 0.21904169259489734,
+      "grad_norm": 3.7298269271850586,
+      "learning_rate": 3.360995850622407e-06,
+      "loss": 1.8717,
+      "step": 176
+    },
+    {
+      "epoch": 0.22028624766645924,
+      "grad_norm": 3.422295093536377,
+      "learning_rate": 3.381742738589212e-06,
+      "loss": 1.8568,
+      "step": 177
+    },
+    {
+      "epoch": 0.22153080273802117,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.8471,
+      "step": 178
+    },
+    {
+      "epoch": 0.22277535780958307,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.9303,
+      "step": 179
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "grad_norm": 3.506223440170288,
+      "learning_rate": 3.423236514522822e-06,
+      "loss": 1.8237,
+      "step": 180
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "eval_loss": 1.9734643697738647,
+      "eval_runtime": 52.2815,
+      "eval_samples_per_second": 19.127,
+      "eval_steps_per_second": 0.803,
+      "step": 180
+    },
+    {
+      "epoch": 0.2252644679527069,
+      "grad_norm": 3.0156619548797607,
+      "learning_rate": 3.4439834024896267e-06,
+      "loss": 1.908,
+      "step": 181
+    },
+    {
+      "epoch": 0.2265090230242688,
+      "grad_norm": 3.55517315864563,
+      "learning_rate": 3.4647302904564318e-06,
+      "loss": 1.9104,
+      "step": 182
+    },
+    {
+      "epoch": 0.22775357809583074,
+      "grad_norm": 3.144984006881714,
+      "learning_rate": 3.4854771784232365e-06,
+      "loss": 1.8198,
+      "step": 183
+    },
+    {
+      "epoch": 0.22899813316739265,
+      "grad_norm": 4.302074432373047,
+      "learning_rate": 3.5062240663900416e-06,
+      "loss": 1.8237,
+      "step": 184
+    },
+    {
+      "epoch": 0.23024268823895458,
+      "grad_norm": 3.0522446632385254,
+      "learning_rate": 3.526970954356847e-06,
+      "loss": 1.8409,
+      "step": 185
+    },
+    {
+      "epoch": 0.23148724331051648,
+      "grad_norm": 3.8607394695281982,
+      "learning_rate": 3.5477178423236518e-06,
+      "loss": 1.8807,
+      "step": 186
+    },
+    {
+      "epoch": 0.23273179838207841,
+      "grad_norm": 2.9236302375793457,
+      "learning_rate": 3.568464730290457e-06,
+      "loss": 1.7873,
+      "step": 187
+    },
+    {
+      "epoch": 0.23397635345364032,
+      "grad_norm": 4.013780117034912,
+      "learning_rate": 3.5892116182572616e-06,
+      "loss": 1.7909,
+      "step": 188
+    },
+    {
+      "epoch": 0.23522090852520225,
+      "grad_norm": 3.0933122634887695,
+      "learning_rate": 3.6099585062240667e-06,
+      "loss": 1.8469,
+      "step": 189
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "grad_norm": 3.487816095352173,
+      "learning_rate": 3.6307053941908714e-06,
+      "loss": 1.8469,
+      "step": 190
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "eval_loss": 1.9604240655899048,
+      "eval_runtime": 53.4604,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 0.786,
+      "step": 190
+    },
+    {
+      "epoch": 0.23771001866832608,
+      "grad_norm": 2.829219341278076,
+      "learning_rate": 3.6514522821576765e-06,
+      "loss": 1.8639,
+      "step": 191
+    },
+    {
+      "epoch": 0.238954573739888,
+      "grad_norm": 3.595534324645996,
+      "learning_rate": 3.672199170124482e-06,
+      "loss": 1.8988,
+      "step": 192
+    },
+    {
+      "epoch": 0.24019912881144992,
+      "grad_norm": 2.9022483825683594,
+      "learning_rate": 3.6929460580912867e-06,
+      "loss": 1.8972,
+      "step": 193
+    },
+    {
+      "epoch": 0.24144368388301182,
+      "grad_norm": 3.0667994022369385,
+      "learning_rate": 3.713692946058092e-06,
+      "loss": 1.8499,
+      "step": 194
+    },
+    {
+      "epoch": 0.24268823895457373,
+      "grad_norm": 3.0498485565185547,
+      "learning_rate": 3.7344398340248965e-06,
+      "loss": 1.8341,
+      "step": 195
+    },
+    {
+      "epoch": 0.24393279402613566,
+      "grad_norm": 4.058084964752197,
+      "learning_rate": 3.7551867219917016e-06,
+      "loss": 1.8641,
+      "step": 196
+    },
+    {
+      "epoch": 0.24517734909769756,
+      "grad_norm": 2.9995028972625732,
+      "learning_rate": 3.7759336099585063e-06,
+      "loss": 1.7885,
+      "step": 197
+    },
+    {
+      "epoch": 0.2464219041692595,
+      "grad_norm": 3.254704475402832,
+      "learning_rate": 3.7966804979253114e-06,
+      "loss": 1.7544,
+      "step": 198
+    },
+    {
+      "epoch": 0.2476664592408214,
+      "grad_norm": 2.9126744270324707,
+      "learning_rate": 3.817427385892116e-06,
+      "loss": 1.8461,
+      "step": 199
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "grad_norm": 3.475247621536255,
+      "learning_rate": 3.838174273858922e-06,
+      "loss": 1.8095,
+      "step": 200
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "eval_loss": 1.9373760223388672,
+      "eval_runtime": 51.8733,
+      "eval_samples_per_second": 19.278,
+      "eval_steps_per_second": 0.81,
+      "step": 200
+    },
+    {
+      "epoch": 0.25015556938394523,
+      "grad_norm": 3.2663731575012207,
+      "learning_rate": 3.858921161825726e-06,
+      "loss": 1.8076,
+      "step": 201
+    },
+    {
+      "epoch": 0.25140012445550713,
+      "grad_norm": 3.1151726245880127,
+      "learning_rate": 3.879668049792531e-06,
+      "loss": 1.8143,
+      "step": 202
+    },
+    {
+      "epoch": 0.2526446795270691,
+      "grad_norm": 3.102038860321045,
+      "learning_rate": 3.9004149377593365e-06,
+      "loss": 1.8279,
+      "step": 203
+    },
+    {
+      "epoch": 0.253889234598631,
+      "grad_norm": 3.370642900466919,
+      "learning_rate": 3.921161825726142e-06,
+      "loss": 1.8522,
+      "step": 204
+    },
+    {
+      "epoch": 0.2551337896701929,
+      "grad_norm": 3.116128921508789,
+      "learning_rate": 3.941908713692946e-06,
+      "loss": 1.8104,
+      "step": 205
+    },
+    {
+      "epoch": 0.2563783447417548,
+      "grad_norm": 3.12028431892395,
+      "learning_rate": 3.962655601659751e-06,
+      "loss": 1.7723,
+      "step": 206
+    },
+    {
+      "epoch": 0.25762289981331676,
+      "grad_norm": 3.1251418590545654,
+      "learning_rate": 3.983402489626556e-06,
+      "loss": 1.863,
+      "step": 207
+    },
+    {
+      "epoch": 0.25886745488487867,
+      "grad_norm": 3.073702812194824,
+      "learning_rate": 4.004149377593361e-06,
+      "loss": 1.805,
+      "step": 208
+    },
+    {
+      "epoch": 0.26011200995644057,
+      "grad_norm": 2.9772469997406006,
+      "learning_rate": 4.024896265560166e-06,
+      "loss": 1.8251,
+      "step": 209
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "grad_norm": 2.9970037937164307,
+      "learning_rate": 4.045643153526971e-06,
+      "loss": 1.8386,
+      "step": 210
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "eval_loss": 1.9167065620422363,
+      "eval_runtime": 42.3328,
+      "eval_samples_per_second": 23.622,
+      "eval_steps_per_second": 0.992,
+      "step": 210
+    },
+    {
+      "epoch": 0.26260112009956443,
+      "grad_norm": 3.1391561031341553,
+      "learning_rate": 4.0663900414937765e-06,
+      "loss": 1.8109,
+      "step": 211
+    },
+    {
+      "epoch": 0.26384567517112634,
+      "grad_norm": 3.161538600921631,
+      "learning_rate": 4.087136929460581e-06,
+      "loss": 1.8441,
+      "step": 212
+    },
+    {
+      "epoch": 0.26509023024268824,
+      "grad_norm": 3.12412166595459,
+      "learning_rate": 4.107883817427386e-06,
+      "loss": 1.7736,
+      "step": 213
+    },
+    {
+      "epoch": 0.26633478531425014,
+      "grad_norm": 3.3241145610809326,
+      "learning_rate": 4.128630705394191e-06,
+      "loss": 1.8067,
+      "step": 214
+    },
+    {
+      "epoch": 0.26757934038581205,
+      "grad_norm": 3.503307819366455,
+      "learning_rate": 4.149377593360996e-06,
+      "loss": 1.7858,
+      "step": 215
+    },
+    {
+      "epoch": 0.268823895457374,
+      "grad_norm": 3.1009578704833984,
+      "learning_rate": 4.170124481327801e-06,
+      "loss": 1.7542,
+      "step": 216
+    },
+    {
+      "epoch": 0.2700684505289359,
+      "grad_norm": 3.2236011028289795,
+      "learning_rate": 4.190871369294606e-06,
+      "loss": 1.8006,
+      "step": 217
+    },
+    {
+      "epoch": 0.2713130056004978,
+      "grad_norm": 3.048935651779175,
+      "learning_rate": 4.211618257261411e-06,
+      "loss": 1.7706,
+      "step": 218
+    },
+    {
+      "epoch": 0.2725575606720597,
+      "grad_norm": 3.3416147232055664,
+      "learning_rate": 4.232365145228216e-06,
+      "loss": 1.8108,
+      "step": 219
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "grad_norm": 3.2010738849639893,
+      "learning_rate": 4.253112033195021e-06,
+      "loss": 1.8561,
+      "step": 220
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "eval_loss": 1.9111930131912231,
+      "eval_runtime": 50.4077,
+      "eval_samples_per_second": 19.838,
+      "eval_steps_per_second": 0.833,
+      "step": 220
+    },
+    {
+      "epoch": 0.2750466708151836,
+      "grad_norm": 3.4033374786376953,
+      "learning_rate": 4.273858921161826e-06,
+      "loss": 1.8218,
+      "step": 221
+    },
+    {
+      "epoch": 0.2762912258867455,
+      "grad_norm": 3.2086002826690674,
+      "learning_rate": 4.294605809128631e-06,
+      "loss": 1.8213,
+      "step": 222
+    },
+    {
+      "epoch": 0.2775357809583074,
+      "grad_norm": 4.090956211090088,
+      "learning_rate": 4.315352697095436e-06,
+      "loss": 1.8455,
+      "step": 223
+    },
+    {
+      "epoch": 0.27878033602986935,
+      "grad_norm": 3.1142985820770264,
+      "learning_rate": 4.336099585062241e-06,
+      "loss": 1.7737,
+      "step": 224
+    },
+    {
+      "epoch": 0.28002489110143125,
+      "grad_norm": 3.369669198989868,
+      "learning_rate": 4.356846473029046e-06,
+      "loss": 1.824,
+      "step": 225
+    },
+    {
+      "epoch": 0.28126944617299315,
+      "grad_norm": 3.165672779083252,
+      "learning_rate": 4.3775933609958506e-06,
+      "loss": 1.7768,
+      "step": 226
+    },
+    {
+      "epoch": 0.28251400124455506,
+      "grad_norm": 3.3343470096588135,
+      "learning_rate": 4.398340248962656e-06,
+      "loss": 1.7278,
+      "step": 227
+    },
+    {
+      "epoch": 0.28375855631611696,
+      "grad_norm": 3.2574994564056396,
+      "learning_rate": 4.419087136929461e-06,
+      "loss": 1.8439,
+      "step": 228
+    },
+    {
+      "epoch": 0.2850031113876789,
+      "grad_norm": 3.043928623199463,
+      "learning_rate": 4.439834024896266e-06,
+      "loss": 1.818,
+      "step": 229
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "grad_norm": 2.9703125953674316,
+      "learning_rate": 4.460580912863071e-06,
+      "loss": 1.773,
+      "step": 230
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "eval_loss": 1.898109793663025,
+      "eval_runtime": 45.6023,
+      "eval_samples_per_second": 21.929,
+      "eval_steps_per_second": 0.921,
+      "step": 230
+    },
+    {
+      "epoch": 0.2874922215308027,
+      "grad_norm": 2.9997220039367676,
+      "learning_rate": 4.481327800829876e-06,
+      "loss": 1.7727,
+      "step": 231
+    },
+    {
+      "epoch": 0.28873677660236463,
+      "grad_norm": 3.1358482837677,
+      "learning_rate": 4.502074688796681e-06,
+      "loss": 1.7764,
+      "step": 232
+    },
+    {
+      "epoch": 0.2899813316739266,
+      "grad_norm": 3.395747661590576,
+      "learning_rate": 4.5228215767634855e-06,
+      "loss": 1.7826,
+      "step": 233
+    },
+    {
+      "epoch": 0.2912258867454885,
+      "grad_norm": 3.7494754791259766,
+      "learning_rate": 4.543568464730291e-06,
+      "loss": 1.7181,
+      "step": 234
+    },
+    {
+      "epoch": 0.2924704418170504,
+      "grad_norm": 3.42293381690979,
+      "learning_rate": 4.564315352697096e-06,
+      "loss": 1.7773,
+      "step": 235
+    },
+    {
+      "epoch": 0.2937149968886123,
+      "grad_norm": 3.2524514198303223,
+      "learning_rate": 4.585062240663901e-06,
+      "loss": 1.7294,
+      "step": 236
+    },
+    {
+      "epoch": 0.29495955196017426,
+      "grad_norm": 3.657869577407837,
+      "learning_rate": 4.605809128630706e-06,
+      "loss": 1.762,
+      "step": 237
+    },
+    {
+      "epoch": 0.29620410703173616,
+      "grad_norm": 3.127372980117798,
+      "learning_rate": 4.626556016597511e-06,
+      "loss": 1.7623,
+      "step": 238
+    },
+    {
+      "epoch": 0.29744866210329807,
+      "grad_norm": 4.166962146759033,
+      "learning_rate": 4.647302904564316e-06,
+      "loss": 1.6995,
+      "step": 239
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "grad_norm": 3.094264030456543,
+      "learning_rate": 4.66804979253112e-06,
+      "loss": 1.7469,
+      "step": 240
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "eval_loss": 1.8908178806304932,
+      "eval_runtime": 44.6332,
+      "eval_samples_per_second": 22.405,
+      "eval_steps_per_second": 0.941,
+      "step": 240
+    },
+    {
+      "epoch": 0.29993777224642193,
+      "grad_norm": 3.8644745349884033,
+      "learning_rate": 4.6887966804979255e-06,
+      "loss": 1.7644,
+      "step": 241
+    },
+    {
+      "epoch": 0.30118232731798383,
+      "grad_norm": 3.1488852500915527,
+      "learning_rate": 4.709543568464731e-06,
+      "loss": 1.772,
+      "step": 242
+    },
+    {
+      "epoch": 0.30242688238954574,
+      "grad_norm": 3.3179638385772705,
+      "learning_rate": 4.730290456431536e-06,
+      "loss": 1.7588,
+      "step": 243
+    },
+    {
+      "epoch": 0.30367143746110764,
+      "grad_norm": 3.328355550765991,
+      "learning_rate": 4.751037344398341e-06,
+      "loss": 1.7417,
+      "step": 244
+    },
+    {
+      "epoch": 0.30491599253266954,
+      "grad_norm": 3.2690482139587402,
+      "learning_rate": 4.771784232365146e-06,
+      "loss": 1.7562,
+      "step": 245
+    },
+    {
+      "epoch": 0.3061605476042315,
+      "grad_norm": 3.2759454250335693,
+      "learning_rate": 4.792531120331951e-06,
+      "loss": 1.7536,
+      "step": 246
+    },
+    {
+      "epoch": 0.3074051026757934,
+      "grad_norm": 2.937964916229248,
+      "learning_rate": 4.813278008298755e-06,
+      "loss": 1.7518,
+      "step": 247
+    },
+    {
+      "epoch": 0.3086496577473553,
+      "grad_norm": 3.0995302200317383,
+      "learning_rate": 4.83402489626556e-06,
+      "loss": 1.7593,
+      "step": 248
+    },
+    {
+      "epoch": 0.3098942128189172,
+      "grad_norm": 3.230459213256836,
+      "learning_rate": 4.8547717842323655e-06,
+      "loss": 1.6962,
+      "step": 249
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "grad_norm": 3.242576837539673,
+      "learning_rate": 4.875518672199171e-06,
+      "loss": 1.7307,
+      "step": 250
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "eval_loss": 1.8805371522903442,
+      "eval_runtime": 45.9886,
+      "eval_samples_per_second": 21.745,
+      "eval_steps_per_second": 0.913,
+      "step": 250
+    },
+    {
+      "epoch": 0.3123833229620411,
+      "grad_norm": 3.097045421600342,
+      "learning_rate": 4.896265560165976e-06,
+      "loss": 1.7438,
+      "step": 251
+    },
+    {
+      "epoch": 0.313627878033603,
+      "grad_norm": 3.2428948879241943,
+      "learning_rate": 4.91701244813278e-06,
+      "loss": 1.7179,
+      "step": 252
+    },
+    {
+      "epoch": 0.3148724331051649,
+      "grad_norm": 3.196274518966675,
+      "learning_rate": 4.937759336099586e-06,
+      "loss": 1.7855,
+      "step": 253
+    },
+    {
+      "epoch": 0.31611698817672684,
+      "grad_norm": 2.978203535079956,
+      "learning_rate": 4.95850622406639e-06,
+      "loss": 1.7144,
+      "step": 254
+    },
+    {
+      "epoch": 0.31736154324828875,
+      "grad_norm": 3.2641701698303223,
+      "learning_rate": 4.979253112033195e-06,
+      "loss": 1.7427,
+      "step": 255
+    },
+    {
+      "epoch": 0.31860609831985065,
+      "grad_norm": 2.7441232204437256,
+      "learning_rate": 5e-06,
+      "loss": 1.7619,
+      "step": 256
+    },
+    {
+      "epoch": 0.31985065339141255,
+      "grad_norm": 3.0723495483398438,
+      "learning_rate": 4.99769372693727e-06,
+      "loss": 1.8017,
+      "step": 257
+    },
+    {
+      "epoch": 0.3210952084629745,
+      "grad_norm": 3.104752540588379,
+      "learning_rate": 4.995387453874539e-06,
+      "loss": 1.7761,
+      "step": 258
+    },
+    {
+      "epoch": 0.3223397635345364,
+      "grad_norm": 3.138627052307129,
+      "learning_rate": 4.993081180811809e-06,
+      "loss": 1.7879,
+      "step": 259
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "grad_norm": 2.8567333221435547,
+      "learning_rate": 4.990774907749078e-06,
+      "loss": 1.7561,
+      "step": 260
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "eval_loss": 1.8606494665145874,
+      "eval_runtime": 46.453,
+      "eval_samples_per_second": 21.527,
+      "eval_steps_per_second": 0.904,
+      "step": 260
+    },
+    {
+      "epoch": 0.3248288736776602,
+      "grad_norm": 3.2696540355682373,
+      "learning_rate": 4.988468634686347e-06,
+      "loss": 1.7201,
+      "step": 261
+    },
+    {
+      "epoch": 0.3260734287492221,
+      "grad_norm": 2.6692731380462646,
+      "learning_rate": 4.986162361623617e-06,
+      "loss": 1.7264,
+      "step": 262
+    },
+    {
+      "epoch": 0.3273179838207841,
+      "grad_norm": 3.047549247741699,
+      "learning_rate": 4.983856088560886e-06,
+      "loss": 1.7362,
+      "step": 263
+    },
+    {
+      "epoch": 0.328562538892346,
+      "grad_norm": 3.0667457580566406,
+      "learning_rate": 4.981549815498156e-06,
+      "loss": 1.7321,
+      "step": 264
+    },
+    {
+      "epoch": 0.3298070939639079,
+      "grad_norm": 2.560047149658203,
+      "learning_rate": 4.979243542435424e-06,
+      "loss": 1.7508,
+      "step": 265
+    },
+    {
+      "epoch": 0.3310516490354698,
+      "grad_norm": 3.26595401763916,
+      "learning_rate": 4.976937269372694e-06,
+      "loss": 1.7248,
+      "step": 266
+    },
+    {
+      "epoch": 0.33229620410703176,
+      "grad_norm": 2.929210662841797,
+      "learning_rate": 4.974630996309964e-06,
+      "loss": 1.697,
+      "step": 267
+    },
+    {
+      "epoch": 0.33354075917859366,
+      "grad_norm": 3.1526286602020264,
+      "learning_rate": 4.972324723247233e-06,
+      "loss": 1.7385,
+      "step": 268
+    },
+    {
+      "epoch": 0.33478531425015556,
+      "grad_norm": 3.001619577407837,
+      "learning_rate": 4.970018450184502e-06,
+      "loss": 1.7072,
+      "step": 269
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "grad_norm": 3.2464189529418945,
+      "learning_rate": 4.9677121771217715e-06,
+      "loss": 1.7361,
+      "step": 270
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "eval_loss": 1.860226035118103,
+      "eval_runtime": 46.6881,
+      "eval_samples_per_second": 21.419,
+      "eval_steps_per_second": 0.9,
+      "step": 270
+    },
+    {
+      "epoch": 0.3372744243932794,
+      "grad_norm": 3.2104530334472656,
+      "learning_rate": 4.965405904059041e-06,
+      "loss": 1.7171,
+      "step": 271
+    },
+    {
+      "epoch": 0.33851897946484133,
+      "grad_norm": 3.146847724914551,
+      "learning_rate": 4.96309963099631e-06,
+      "loss": 1.7311,
+      "step": 272
+    },
+    {
+      "epoch": 0.33976353453640323,
+      "grad_norm": 3.1431286334991455,
+      "learning_rate": 4.96079335793358e-06,
+      "loss": 1.7345,
+      "step": 273
+    },
+    {
+      "epoch": 0.34100808960796514,
+      "grad_norm": 2.847163677215576,
+      "learning_rate": 4.958487084870849e-06,
+      "loss": 1.7262,
+      "step": 274
+    },
+    {
+      "epoch": 0.3422526446795271,
+      "grad_norm": 3.1987810134887695,
+      "learning_rate": 4.956180811808119e-06,
+      "loss": 1.7945,
+      "step": 275
+    },
+    {
+      "epoch": 0.343497199751089,
+      "grad_norm": 2.7475385665893555,
+      "learning_rate": 4.953874538745388e-06,
+      "loss": 1.76,
+      "step": 276
+    },
+    {
+      "epoch": 0.3447417548226509,
+      "grad_norm": 3.0427663326263428,
+      "learning_rate": 4.9515682656826574e-06,
+      "loss": 1.7324,
+      "step": 277
+    },
+    {
+      "epoch": 0.3459863098942128,
+      "grad_norm": 3.605212688446045,
+      "learning_rate": 4.949261992619927e-06,
+      "loss": 1.7712,
+      "step": 278
+    },
+    {
+      "epoch": 0.3472308649657747,
+      "grad_norm": 3.0564935207366943,
+      "learning_rate": 4.946955719557196e-06,
+      "loss": 1.6792,
+      "step": 279
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "grad_norm": 3.1069741249084473,
+      "learning_rate": 4.944649446494466e-06,
+      "loss": 1.7145,
+      "step": 280
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "eval_loss": 1.8537051677703857,
+      "eval_runtime": 47.4813,
+      "eval_samples_per_second": 21.061,
+      "eval_steps_per_second": 0.885,
+      "step": 280
+    },
+    {
+      "epoch": 0.3497199751088986,
+      "grad_norm": 2.8021512031555176,
+      "learning_rate": 4.942343173431734e-06,
+      "loss": 1.7145,
+      "step": 281
+    },
+    {
+      "epoch": 0.3509645301804605,
+      "grad_norm": 3.5549023151397705,
+      "learning_rate": 4.940036900369004e-06,
+      "loss": 1.7642,
+      "step": 282
+    },
+    {
+      "epoch": 0.3522090852520224,
+      "grad_norm": 2.7648985385894775,
+      "learning_rate": 4.937730627306274e-06,
+      "loss": 1.7255,
+      "step": 283
+    },
+    {
+      "epoch": 0.35345364032358434,
+      "grad_norm": 3.0815863609313965,
+      "learning_rate": 4.9354243542435426e-06,
+      "loss": 1.7055,
+      "step": 284
+    },
+    {
+      "epoch": 0.35469819539514624,
+      "grad_norm": 2.9009227752685547,
+      "learning_rate": 4.933118081180812e-06,
+      "loss": 1.7019,
+      "step": 285
+    },
+    {
+      "epoch": 0.35594275046670815,
+      "grad_norm": 2.618429660797119,
+      "learning_rate": 4.930811808118081e-06,
+      "loss": 1.6318,
+      "step": 286
+    },
+    {
+      "epoch": 0.35718730553827005,
+      "grad_norm": 3.368230104446411,
+      "learning_rate": 4.928505535055351e-06,
+      "loss": 1.7304,
+      "step": 287
+    },
+    {
+      "epoch": 0.358431860609832,
+      "grad_norm": 3.256889820098877,
+      "learning_rate": 4.92619926199262e-06,
+      "loss": 1.7019,
+      "step": 288
+    },
+    {
+      "epoch": 0.3596764156813939,
+      "grad_norm": 2.9366304874420166,
+      "learning_rate": 4.92389298892989e-06,
+      "loss": 1.734,
+      "step": 289
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "grad_norm": 3.0393142700195312,
+      "learning_rate": 4.921586715867159e-06,
+      "loss": 1.7504,
+      "step": 290
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "eval_loss": 1.8359886407852173,
+      "eval_runtime": 49.7622,
+      "eval_samples_per_second": 20.096,
+      "eval_steps_per_second": 0.844,
+      "step": 290
+    },
+    {
+      "epoch": 0.3621655258245177,
+      "grad_norm": 2.7288589477539062,
+      "learning_rate": 4.9192804428044285e-06,
+      "loss": 1.7004,
+      "step": 291
+    },
+    {
+      "epoch": 0.3634100808960797,
+      "grad_norm": 3.2793378829956055,
+      "learning_rate": 4.916974169741698e-06,
+      "loss": 1.7028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3646546359676416,
+      "grad_norm": 2.960880756378174,
+      "learning_rate": 4.914667896678967e-06,
+      "loss": 1.6759,
+      "step": 293
+    },
+    {
+      "epoch": 0.3658991910392035,
+      "grad_norm": 2.836421012878418,
+      "learning_rate": 4.912361623616237e-06,
+      "loss": 1.6475,
+      "step": 294
+    },
+    {
+      "epoch": 0.3671437461107654,
+      "grad_norm": 2.9495465755462646,
+      "learning_rate": 4.910055350553506e-06,
+      "loss": 1.7076,
+      "step": 295
+    },
+    {
+      "epoch": 0.3683883011823273,
+      "grad_norm": 2.954730272293091,
+      "learning_rate": 4.907749077490776e-06,
+      "loss": 1.7261,
+      "step": 296
+    },
+    {
+      "epoch": 0.36963285625388925,
+      "grad_norm": 2.8167543411254883,
+      "learning_rate": 4.905442804428044e-06,
+      "loss": 1.6939,
+      "step": 297
+    },
+    {
+      "epoch": 0.37087741132545116,
+      "grad_norm": 2.857316017150879,
+      "learning_rate": 4.903136531365314e-06,
+      "loss": 1.7362,
+      "step": 298
+    },
+    {
+      "epoch": 0.37212196639701306,
+      "grad_norm": 2.8612918853759766,
+      "learning_rate": 4.900830258302584e-06,
+      "loss": 1.6489,
+      "step": 299
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "grad_norm": 2.9618914127349854,
+      "learning_rate": 4.898523985239853e-06,
+      "loss": 1.718,
+      "step": 300
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "eval_loss": 1.827571988105774,
+      "eval_runtime": 46.4552,
+      "eval_samples_per_second": 21.526,
+      "eval_steps_per_second": 0.904,
+      "step": 300
+    },
+    {
+      "epoch": 0.3746110765401369,
+      "grad_norm": 2.8706953525543213,
+      "learning_rate": 4.896217712177122e-06,
+      "loss": 1.6925,
+      "step": 301
+    },
+    {
+      "epoch": 0.3758556316116988,
+      "grad_norm": 3.2565090656280518,
+      "learning_rate": 4.893911439114391e-06,
+      "loss": 1.7369,
+      "step": 302
+    },
+    {
+      "epoch": 0.37710018668326073,
+      "grad_norm": 2.928858995437622,
+      "learning_rate": 4.891605166051661e-06,
+      "loss": 1.6882,
+      "step": 303
+    },
+    {
+      "epoch": 0.37834474175482263,
+      "grad_norm": 3.042314052581787,
+      "learning_rate": 4.88929889298893e-06,
+      "loss": 1.6466,
+      "step": 304
+    },
+    {
+      "epoch": 0.3795892968263846,
+      "grad_norm": 2.7509143352508545,
+      "learning_rate": 4.8869926199262e-06,
+      "loss": 1.685,
+      "step": 305
+    },
+    {
+      "epoch": 0.3808338518979465,
+      "grad_norm": 2.9601588249206543,
+      "learning_rate": 4.884686346863469e-06,
+      "loss": 1.6772,
+      "step": 306
+    },
+    {
+      "epoch": 0.3820784069695084,
+      "grad_norm": 2.618608236312866,
+      "learning_rate": 4.8823800738007384e-06,
+      "loss": 1.7127,
+      "step": 307
+    },
+    {
+      "epoch": 0.3833229620410703,
+      "grad_norm": 3.047581195831299,
+      "learning_rate": 4.880073800738008e-06,
+      "loss": 1.7174,
+      "step": 308
+    },
+    {
+      "epoch": 0.3845675171126322,
+      "grad_norm": 2.87735652923584,
+      "learning_rate": 4.877767527675277e-06,
+      "loss": 1.6677,
+      "step": 309
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "grad_norm": 2.7169721126556396,
+      "learning_rate": 4.875461254612546e-06,
+      "loss": 1.7486,
+      "step": 310
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "eval_loss": 1.8221794366836548,
+      "eval_runtime": 43.1681,
+      "eval_samples_per_second": 23.165,
+      "eval_steps_per_second": 0.973,
+      "step": 310
+    },
+    {
+      "epoch": 0.38705662725575607,
+      "grad_norm": 2.7094991207122803,
+      "learning_rate": 4.873154981549816e-06,
+      "loss": 1.7348,
+      "step": 311
+    },
+    {
+      "epoch": 0.388301182327318,
+      "grad_norm": 2.8989078998565674,
+      "learning_rate": 4.8708487084870856e-06,
+      "loss": 1.7352,
+      "step": 312
+    },
+    {
+      "epoch": 0.3895457373988799,
+      "grad_norm": 2.9565601348876953,
+      "learning_rate": 4.868542435424355e-06,
+      "loss": 1.7063,
+      "step": 313
+    },
+    {
+      "epoch": 0.39079029247044184,
+      "grad_norm": 2.8354082107543945,
+      "learning_rate": 4.8662361623616235e-06,
+      "loss": 1.6808,
+      "step": 314
+    },
+    {
+      "epoch": 0.39203484754200374,
+      "grad_norm": 2.8287479877471924,
+      "learning_rate": 4.863929889298894e-06,
+      "loss": 1.6731,
+      "step": 315
+    },
+    {
+      "epoch": 0.39327940261356564,
+      "grad_norm": 2.6537821292877197,
+      "learning_rate": 4.861623616236163e-06,
+      "loss": 1.695,
+      "step": 316
+    },
+    {
+      "epoch": 0.39452395768512755,
+      "grad_norm": 2.7135190963745117,
+      "learning_rate": 4.859317343173432e-06,
+      "loss": 1.6897,
+      "step": 317
+    },
+    {
+      "epoch": 0.3957685127566895,
+      "grad_norm": 3.1019129753112793,
+      "learning_rate": 4.857011070110701e-06,
+      "loss": 1.7219,
+      "step": 318
+    },
+    {
+      "epoch": 0.3970130678282514,
+      "grad_norm": 2.945737838745117,
+      "learning_rate": 4.854704797047971e-06,
+      "loss": 1.7316,
+      "step": 319
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "grad_norm": 2.7668251991271973,
+      "learning_rate": 4.85239852398524e-06,
+      "loss": 1.6461,
+      "step": 320
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "eval_loss": 1.8210570812225342,
+      "eval_runtime": 46.2552,
+      "eval_samples_per_second": 21.619,
+      "eval_steps_per_second": 0.908,
+      "step": 320
+    },
+    {
+      "epoch": 0.3995021779713752,
+      "grad_norm": 2.738109588623047,
+      "learning_rate": 4.8500922509225095e-06,
+      "loss": 1.6907,
+      "step": 321
+    },
+    {
+      "epoch": 0.4007467330429372,
+      "grad_norm": 3.0195062160491943,
+      "learning_rate": 4.847785977859779e-06,
+      "loss": 1.7099,
+      "step": 322
+    },
+    {
+      "epoch": 0.4019912881144991,
+      "grad_norm": 2.8672404289245605,
+      "learning_rate": 4.845479704797048e-06,
+      "loss": 1.6509,
+      "step": 323
+    },
+    {
+      "epoch": 0.403235843186061,
+      "grad_norm": 2.9041008949279785,
+      "learning_rate": 4.843173431734318e-06,
+      "loss": 1.5818,
+      "step": 324
+    },
+    {
+      "epoch": 0.4044803982576229,
+      "grad_norm": 2.9105985164642334,
+      "learning_rate": 4.840867158671587e-06,
+      "loss": 1.7002,
+      "step": 325
+    },
+    {
+      "epoch": 0.4057249533291848,
+      "grad_norm": 2.749009847640991,
+      "learning_rate": 4.838560885608857e-06,
+      "loss": 1.7021,
+      "step": 326
+    },
+    {
+      "epoch": 0.40696950840074675,
+      "grad_norm": 2.6998822689056396,
+      "learning_rate": 4.836254612546126e-06,
+      "loss": 1.6853,
+      "step": 327
+    },
+    {
+      "epoch": 0.40821406347230865,
+      "grad_norm": 2.678201675415039,
+      "learning_rate": 4.8339483394833955e-06,
+      "loss": 1.6934,
+      "step": 328
+    },
+    {
+      "epoch": 0.40945861854387056,
+      "grad_norm": 2.774845600128174,
+      "learning_rate": 4.831642066420665e-06,
+      "loss": 1.657,
+      "step": 329
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "grad_norm": 2.6363000869750977,
+      "learning_rate": 4.8293357933579335e-06,
+      "loss": 1.6555,
+      "step": 330
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "eval_loss": 1.8126581907272339,
+      "eval_runtime": 52.7717,
+      "eval_samples_per_second": 18.95,
+      "eval_steps_per_second": 0.796,
+      "step": 330
+    },
+    {
+      "epoch": 0.4119477286869944,
+      "grad_norm": 3.011659622192383,
+      "learning_rate": 4.827029520295204e-06,
+      "loss": 1.7202,
+      "step": 331
+    },
+    {
+      "epoch": 0.4131922837585563,
+      "grad_norm": 2.8322594165802,
+      "learning_rate": 4.824723247232473e-06,
+      "loss": 1.6736,
+      "step": 332
+    },
+    {
+      "epoch": 0.4144368388301182,
+      "grad_norm": 2.9360146522521973,
+      "learning_rate": 4.822416974169742e-06,
+      "loss": 1.7213,
+      "step": 333
+    },
+    {
+      "epoch": 0.41568139390168013,
+      "grad_norm": 2.810206413269043,
+      "learning_rate": 4.820110701107011e-06,
+      "loss": 1.6867,
+      "step": 334
+    },
+    {
+      "epoch": 0.4169259489732421,
+      "grad_norm": 2.6864097118377686,
+      "learning_rate": 4.817804428044281e-06,
+      "loss": 1.6854,
+      "step": 335
+    },
+    {
+      "epoch": 0.418170504044804,
+      "grad_norm": 2.7396061420440674,
+      "learning_rate": 4.81549815498155e-06,
+      "loss": 1.68,
+      "step": 336
+    },
+    {
+      "epoch": 0.4194150591163659,
+      "grad_norm": 2.602771520614624,
+      "learning_rate": 4.8131918819188194e-06,
+      "loss": 1.6269,
+      "step": 337
+    },
+    {
+      "epoch": 0.4206596141879278,
+      "grad_norm": 2.8174757957458496,
+      "learning_rate": 4.810885608856089e-06,
+      "loss": 1.6981,
+      "step": 338
+    },
+    {
+      "epoch": 0.42190416925948976,
+      "grad_norm": 3.2398130893707275,
+      "learning_rate": 4.808579335793358e-06,
+      "loss": 1.7061,
+      "step": 339
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "grad_norm": 2.7119736671447754,
+      "learning_rate": 4.806273062730628e-06,
+      "loss": 1.6875,
+      "step": 340
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "eval_loss": 1.7973068952560425,
+      "eval_runtime": 53.5791,
+      "eval_samples_per_second": 18.664,
+      "eval_steps_per_second": 0.784,
+      "step": 340
+    },
+    {
+      "epoch": 0.42439327940261357,
+      "grad_norm": 2.868533134460449,
+      "learning_rate": 4.803966789667897e-06,
+      "loss": 1.6967,
+      "step": 341
+    },
+    {
+      "epoch": 0.42563783447417547,
+      "grad_norm": 2.528083562850952,
+      "learning_rate": 4.8016605166051665e-06,
+      "loss": 1.6195,
+      "step": 342
+    },
+    {
+      "epoch": 0.4268823895457374,
+      "grad_norm": 2.7885115146636963,
+      "learning_rate": 4.799354243542436e-06,
+      "loss": 1.6197,
+      "step": 343
+    },
+    {
+      "epoch": 0.42812694461729933,
+      "grad_norm": 2.768247604370117,
+      "learning_rate": 4.797047970479705e-06,
+      "loss": 1.6271,
+      "step": 344
+    },
+    {
+      "epoch": 0.42937149968886124,
+      "grad_norm": 2.7710275650024414,
+      "learning_rate": 4.794741697416975e-06,
+      "loss": 1.6768,
+      "step": 345
+    },
+    {
+      "epoch": 0.43061605476042314,
+      "grad_norm": 2.968337297439575,
+      "learning_rate": 4.792435424354243e-06,
+      "loss": 1.6433,
+      "step": 346
+    },
+    {
+      "epoch": 0.43186060983198504,
+      "grad_norm": 2.5739808082580566,
+      "learning_rate": 4.790129151291514e-06,
+      "loss": 1.6327,
+      "step": 347
+    },
+    {
+      "epoch": 0.433105164903547,
+      "grad_norm": 2.7676408290863037,
+      "learning_rate": 4.787822878228783e-06,
+      "loss": 1.6178,
+      "step": 348
+    },
+    {
+      "epoch": 0.4343497199751089,
+      "grad_norm": 2.959059476852417,
+      "learning_rate": 4.7855166051660525e-06,
+      "loss": 1.6809,
+      "step": 349
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "grad_norm": 2.559967279434204,
+      "learning_rate": 4.783210332103321e-06,
+      "loss": 1.6704,
+      "step": 350
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "eval_loss": 1.8013949394226074,
+      "eval_runtime": 50.4649,
+      "eval_samples_per_second": 19.816,
+      "eval_steps_per_second": 0.832,
+      "step": 350
+    },
+    {
+      "epoch": 0.4368388301182327,
+      "grad_norm": 2.817901611328125,
+      "learning_rate": 4.7809040590405905e-06,
+      "loss": 1.6296,
+      "step": 351
+    },
+    {
+      "epoch": 0.43808338518979467,
+      "grad_norm": 2.7116055488586426,
+      "learning_rate": 4.778597785977861e-06,
+      "loss": 1.6652,
+      "step": 352
+    },
+    {
+      "epoch": 0.4393279402613566,
+      "grad_norm": 2.544750452041626,
+      "learning_rate": 4.776291512915129e-06,
+      "loss": 1.5668,
+      "step": 353
+    },
+    {
+      "epoch": 0.4405724953329185,
+      "grad_norm": 2.720534086227417,
+      "learning_rate": 4.773985239852399e-06,
+      "loss": 1.6152,
+      "step": 354
+    },
+    {
+      "epoch": 0.4418170504044804,
+      "grad_norm": 2.7270126342773438,
+      "learning_rate": 4.771678966789668e-06,
+      "loss": 1.5624,
+      "step": 355
+    },
+    {
+      "epoch": 0.44306160547604234,
+      "grad_norm": 2.7986624240875244,
+      "learning_rate": 4.769372693726938e-06,
+      "loss": 1.6621,
+      "step": 356
+    },
+    {
+      "epoch": 0.44430616054760425,
+      "grad_norm": 2.943107843399048,
+      "learning_rate": 4.767066420664207e-06,
+      "loss": 1.6713,
+      "step": 357
+    },
+    {
+      "epoch": 0.44555071561916615,
+      "grad_norm": 2.739898681640625,
+      "learning_rate": 4.7647601476014765e-06,
+      "loss": 1.6363,
+      "step": 358
+    },
+    {
+      "epoch": 0.44679527069072805,
+      "grad_norm": 2.8729329109191895,
+      "learning_rate": 4.762453874538746e-06,
+      "loss": 1.6605,
+      "step": 359
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "grad_norm": 2.909029483795166,
+      "learning_rate": 4.760147601476015e-06,
+      "loss": 1.6277,
+      "step": 360
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "eval_loss": 1.8110179901123047,
+      "eval_runtime": 47.924,
+      "eval_samples_per_second": 20.866,
+      "eval_steps_per_second": 0.876,
+      "step": 360
+    },
+    {
+      "epoch": 0.4492843808338519,
+      "grad_norm": 3.3826682567596436,
+      "learning_rate": 4.757841328413285e-06,
+      "loss": 1.7155,
+      "step": 361
+    },
+    {
+      "epoch": 0.4505289359054138,
+      "grad_norm": 3.0586729049682617,
+      "learning_rate": 4.755535055350554e-06,
+      "loss": 1.5991,
+      "step": 362
+    },
+    {
+      "epoch": 0.4517734909769757,
+      "grad_norm": 2.645576238632202,
+      "learning_rate": 4.753228782287823e-06,
+      "loss": 1.6504,
+      "step": 363
+    },
+    {
+      "epoch": 0.4530180460485376,
+      "grad_norm": 2.705967664718628,
+      "learning_rate": 4.750922509225093e-06,
+      "loss": 1.6761,
+      "step": 364
+    },
+    {
+      "epoch": 0.4542626011200996,
+      "grad_norm": 2.7651922702789307,
+      "learning_rate": 4.748616236162362e-06,
+      "loss": 1.61,
+      "step": 365
+    },
+    {
+      "epoch": 0.4555071561916615,
+      "grad_norm": 2.5654966831207275,
+      "learning_rate": 4.746309963099631e-06,
+      "loss": 1.6335,
+      "step": 366
+    },
+    {
+      "epoch": 0.4567517112632234,
+      "grad_norm": 2.6606173515319824,
+      "learning_rate": 4.7440036900369e-06,
+      "loss": 1.6207,
+      "step": 367
+    },
+    {
+      "epoch": 0.4579962663347853,
+      "grad_norm": 2.724755048751831,
+      "learning_rate": 4.741697416974171e-06,
+      "loss": 1.6329,
+      "step": 368
+    },
+    {
+      "epoch": 0.45924082140634725,
+      "grad_norm": 2.7019615173339844,
+      "learning_rate": 4.739391143911439e-06,
+      "loss": 1.6763,
+      "step": 369
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "grad_norm": 2.6594254970550537,
+      "learning_rate": 4.737084870848709e-06,
+      "loss": 1.6174,
+      "step": 370
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "eval_loss": 1.7924141883850098,
+      "eval_runtime": 53.0349,
+      "eval_samples_per_second": 18.855,
+      "eval_steps_per_second": 0.792,
+      "step": 370
+    },
+    {
+      "epoch": 0.46172993154947106,
+      "grad_norm": 2.668421745300293,
+      "learning_rate": 4.734778597785978e-06,
+      "loss": 1.6421,
+      "step": 371
+    },
+    {
+      "epoch": 0.46297448662103297,
+      "grad_norm": 2.7148995399475098,
+      "learning_rate": 4.7324723247232475e-06,
+      "loss": 1.6635,
+      "step": 372
+    },
+    {
+      "epoch": 0.4642190416925949,
+      "grad_norm": 3.156646490097046,
+      "learning_rate": 4.730166051660517e-06,
+      "loss": 1.6679,
+      "step": 373
+    },
+    {
+      "epoch": 0.46546359676415683,
+      "grad_norm": 2.8170645236968994,
+      "learning_rate": 4.727859778597786e-06,
+      "loss": 1.6492,
+      "step": 374
+    },
+    {
+      "epoch": 0.46670815183571873,
+      "grad_norm": 2.9796457290649414,
+      "learning_rate": 4.725553505535056e-06,
+      "loss": 1.6025,
+      "step": 375
+    },
+    {
+      "epoch": 0.46795270690728064,
+      "grad_norm": 2.6493406295776367,
+      "learning_rate": 4.723247232472325e-06,
+      "loss": 1.6398,
+      "step": 376
+    },
+    {
+      "epoch": 0.46919726197884254,
+      "grad_norm": 3.2643542289733887,
+      "learning_rate": 4.720940959409595e-06,
+      "loss": 1.6413,
+      "step": 377
+    },
+    {
+      "epoch": 0.4704418170504045,
+      "grad_norm": 2.7613179683685303,
+      "learning_rate": 4.718634686346864e-06,
+      "loss": 1.6251,
+      "step": 378
+    },
+    {
+      "epoch": 0.4716863721219664,
+      "grad_norm": 3.1216518878936768,
+      "learning_rate": 4.716328413284133e-06,
+      "loss": 1.6268,
+      "step": 379
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "grad_norm": 2.571647882461548,
+      "learning_rate": 4.714022140221403e-06,
+      "loss": 1.6245,
+      "step": 380
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "eval_loss": 1.7875893115997314,
+      "eval_runtime": 49.3058,
+      "eval_samples_per_second": 20.282,
+      "eval_steps_per_second": 0.852,
+      "step": 380
+    },
+    {
+      "epoch": 0.4741754822650902,
+      "grad_norm": 3.443906545639038,
+      "learning_rate": 4.711715867158672e-06,
+      "loss": 1.6799,
+      "step": 381
+    },
+    {
+      "epoch": 0.47542003733665217,
+      "grad_norm": 2.7887957096099854,
+      "learning_rate": 4.709409594095941e-06,
+      "loss": 1.6589,
+      "step": 382
+    },
+    {
+      "epoch": 0.47666459240821407,
+      "grad_norm": 3.2833151817321777,
+      "learning_rate": 4.70710332103321e-06,
+      "loss": 1.6064,
+      "step": 383
+    },
+    {
+      "epoch": 0.477909147479776,
+      "grad_norm": 2.6975057125091553,
+      "learning_rate": 4.704797047970481e-06,
+      "loss": 1.6029,
+      "step": 384
+    },
+    {
+      "epoch": 0.4791537025513379,
+      "grad_norm": 2.745246171951294,
+      "learning_rate": 4.702490774907749e-06,
+      "loss": 1.7127,
+      "step": 385
+    },
+    {
+      "epoch": 0.48039825762289984,
+      "grad_norm": 3.0522775650024414,
+      "learning_rate": 4.700184501845019e-06,
+      "loss": 1.6508,
+      "step": 386
+    },
+    {
+      "epoch": 0.48164281269446174,
+      "grad_norm": 2.689218044281006,
+      "learning_rate": 4.697878228782288e-06,
+      "loss": 1.6971,
+      "step": 387
+    },
+    {
+      "epoch": 0.48288736776602365,
+      "grad_norm": 3.060361862182617,
+      "learning_rate": 4.6955719557195575e-06,
+      "loss": 1.6464,
+      "step": 388
+    },
+    {
+      "epoch": 0.48413192283758555,
+      "grad_norm": 2.872262477874756,
+      "learning_rate": 4.693265682656827e-06,
+      "loss": 1.6293,
+      "step": 389
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "grad_norm": 2.980050563812256,
+      "learning_rate": 4.690959409594096e-06,
+      "loss": 1.6435,
+      "step": 390
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "eval_loss": 1.7791496515274048,
+      "eval_runtime": 50.0299,
+      "eval_samples_per_second": 19.988,
+      "eval_steps_per_second": 0.839,
+      "step": 390
+    },
+    {
+      "epoch": 0.4866210329807094,
+      "grad_norm": 2.764827251434326,
+      "learning_rate": 4.688653136531366e-06,
+      "loss": 1.6279,
+      "step": 391
+    },
+    {
+      "epoch": 0.4878655880522713,
+      "grad_norm": 2.6912333965301514,
+      "learning_rate": 4.686346863468635e-06,
+      "loss": 1.7276,
+      "step": 392
+    },
+    {
+      "epoch": 0.4891101431238332,
+      "grad_norm": 2.8358330726623535,
+      "learning_rate": 4.6840405904059046e-06,
+      "loss": 1.6482,
+      "step": 393
+    },
+    {
+      "epoch": 0.4903546981953951,
+      "grad_norm": 2.6581575870513916,
+      "learning_rate": 4.681734317343174e-06,
+      "loss": 1.6853,
+      "step": 394
+    },
+    {
+      "epoch": 0.4915992532669571,
+      "grad_norm": 2.941425323486328,
+      "learning_rate": 4.6794280442804426e-06,
+      "loss": 1.6075,
+      "step": 395
+    },
+    {
+      "epoch": 0.492843808338519,
+      "grad_norm": 3.2334814071655273,
+      "learning_rate": 4.677121771217713e-06,
+      "loss": 1.6157,
+      "step": 396
+    },
+    {
+      "epoch": 0.4940883634100809,
+      "grad_norm": 2.984410524368286,
+      "learning_rate": 4.674815498154982e-06,
+      "loss": 1.6257,
+      "step": 397
+    },
+    {
+      "epoch": 0.4953329184816428,
+      "grad_norm": 3.0650789737701416,
+      "learning_rate": 4.672509225092252e-06,
+      "loss": 1.6573,
+      "step": 398
+    },
+    {
+      "epoch": 0.49657747355320475,
+      "grad_norm": 2.798664093017578,
+      "learning_rate": 4.67020295202952e-06,
+      "loss": 1.638,
+      "step": 399
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "grad_norm": 2.8266706466674805,
+      "learning_rate": 4.66789667896679e-06,
+      "loss": 1.6484,
+      "step": 400
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "eval_loss": 1.7714862823486328,
+      "eval_runtime": 48.0657,
+      "eval_samples_per_second": 20.805,
+      "eval_steps_per_second": 0.874,
+      "step": 400
+    },
+    {
+      "epoch": 0.49906658369632856,
+      "grad_norm": 2.9928388595581055,
+      "learning_rate": 4.66559040590406e-06,
+      "loss": 1.6721,
+      "step": 401
+    },
+    {
+      "epoch": 0.5003111387678905,
+      "grad_norm": 2.573514461517334,
+      "learning_rate": 4.6632841328413285e-06,
+      "loss": 1.679,
+      "step": 402
+    },
+    {
+      "epoch": 0.5015556938394524,
+      "grad_norm": 3.1736996173858643,
+      "learning_rate": 4.660977859778598e-06,
+      "loss": 1.7124,
+      "step": 403
+    },
+    {
+      "epoch": 0.5028002489110143,
+      "grad_norm": 2.858849287033081,
+      "learning_rate": 4.658671586715867e-06,
+      "loss": 1.6622,
+      "step": 404
+    },
+    {
+      "epoch": 0.5040448039825762,
+      "grad_norm": 2.825698137283325,
+      "learning_rate": 4.656365313653137e-06,
+      "loss": 1.6464,
+      "step": 405
+    },
+    {
+      "epoch": 0.5052893590541382,
+      "grad_norm": 3.783891439437866,
+      "learning_rate": 4.654059040590406e-06,
+      "loss": 1.6665,
+      "step": 406
+    },
+    {
+      "epoch": 0.5065339141257,
+      "grad_norm": 2.789813756942749,
+      "learning_rate": 4.651752767527676e-06,
+      "loss": 1.6064,
+      "step": 407
+    },
+    {
+      "epoch": 0.507778469197262,
+      "grad_norm": 3.3106348514556885,
+      "learning_rate": 4.649446494464945e-06,
+      "loss": 1.6542,
+      "step": 408
+    },
+    {
+      "epoch": 0.5090230242688238,
+      "grad_norm": 2.8805112838745117,
+      "learning_rate": 4.6471402214022145e-06,
+      "loss": 1.6058,
+      "step": 409
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "grad_norm": 3.30898118019104,
+      "learning_rate": 4.644833948339484e-06,
+      "loss": 1.5933,
+      "step": 410
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "eval_loss": 1.7787818908691406,
+      "eval_runtime": 44.0876,
+      "eval_samples_per_second": 22.682,
+      "eval_steps_per_second": 0.953,
+      "step": 410
+    },
+    {
+      "epoch": 0.5115121344119478,
+      "grad_norm": 3.3563625812530518,
+      "learning_rate": 4.642527675276753e-06,
+      "loss": 1.6751,
+      "step": 411
+    },
+    {
+      "epoch": 0.5127566894835096,
+      "grad_norm": 2.968120574951172,
+      "learning_rate": 4.640221402214023e-06,
+      "loss": 1.663,
+      "step": 412
+    },
+    {
+      "epoch": 0.5140012445550716,
+      "grad_norm": 3.310248374938965,
+      "learning_rate": 4.637915129151292e-06,
+      "loss": 1.658,
+      "step": 413
+    },
+    {
+      "epoch": 0.5152457996266335,
+      "grad_norm": 2.7011170387268066,
+      "learning_rate": 4.635608856088562e-06,
+      "loss": 1.6894,
+      "step": 414
+    },
+    {
+      "epoch": 0.5164903546981954,
+      "grad_norm": 2.804901361465454,
+      "learning_rate": 4.63330258302583e-06,
+      "loss": 1.602,
+      "step": 415
+    },
+    {
+      "epoch": 0.5177349097697573,
+      "grad_norm": 3.2412750720977783,
+      "learning_rate": 4.6309963099631e-06,
+      "loss": 1.6158,
+      "step": 416
+    },
+    {
+      "epoch": 0.5189794648413192,
+      "grad_norm": 2.699909210205078,
+      "learning_rate": 4.62869003690037e-06,
+      "loss": 1.5842,
+      "step": 417
+    },
+    {
+      "epoch": 0.5202240199128811,
+      "grad_norm": 3.065929412841797,
+      "learning_rate": 4.6263837638376384e-06,
+      "loss": 1.6434,
+      "step": 418
+    },
+    {
+      "epoch": 0.5214685749844431,
+      "grad_norm": 2.895305633544922,
+      "learning_rate": 4.624077490774908e-06,
+      "loss": 1.5969,
+      "step": 419
+    },
+    {
+      "epoch": 0.522713130056005,
+      "grad_norm": 3.1590664386749268,
+      "learning_rate": 4.621771217712177e-06,
+      "loss": 1.6211,
+      "step": 420
+    },
+    {
+      "epoch": 0.522713130056005,
+      "eval_loss": 1.7674189805984497,
+      "eval_runtime": 53.328,
+      "eval_samples_per_second": 18.752,
+      "eval_steps_per_second": 0.788,
+      "step": 420
+    },
+    {
+      "epoch": 0.5239576851275669,
+      "grad_norm": 2.7126312255859375,
+      "learning_rate": 4.619464944649447e-06,
+      "loss": 1.6246,
+      "step": 421
+    },
+    {
+      "epoch": 0.5252022401991289,
+      "grad_norm": 2.918508768081665,
+      "learning_rate": 4.617158671586716e-06,
+      "loss": 1.6937,
+      "step": 422
+    },
+    {
+      "epoch": 0.5264467952706907,
+      "grad_norm": 3.008610725402832,
+      "learning_rate": 4.6148523985239856e-06,
+      "loss": 1.6864,
+      "step": 423
+    },
+    {
+      "epoch": 0.5276913503422527,
+      "grad_norm": 2.6612043380737305,
+      "learning_rate": 4.612546125461255e-06,
+      "loss": 1.5785,
+      "step": 424
+    },
+    {
+      "epoch": 0.5289359054138145,
+      "grad_norm": 2.850679874420166,
+      "learning_rate": 4.610239852398524e-06,
+      "loss": 1.5899,
+      "step": 425
+    },
+    {
+      "epoch": 0.5301804604853765,
+      "grad_norm": 2.9205031394958496,
+      "learning_rate": 4.607933579335794e-06,
+      "loss": 1.5867,
+      "step": 426
+    },
+    {
+      "epoch": 0.5314250155569384,
+      "grad_norm": 2.929532527923584,
+      "learning_rate": 4.605627306273063e-06,
+      "loss": 1.6182,
+      "step": 427
+    },
+    {
+      "epoch": 0.5326695706285003,
+      "grad_norm": 2.874936103820801,
+      "learning_rate": 4.603321033210333e-06,
+      "loss": 1.5789,
+      "step": 428
+    },
+    {
+      "epoch": 0.5339141257000622,
+      "grad_norm": 2.8703525066375732,
+      "learning_rate": 4.601014760147602e-06,
+      "loss": 1.5859,
+      "step": 429
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "grad_norm": 2.7076902389526367,
+      "learning_rate": 4.5987084870848715e-06,
+      "loss": 1.644,
+      "step": 430
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "eval_loss": 1.7592095136642456,
+      "eval_runtime": 47.9438,
+      "eval_samples_per_second": 20.858,
+      "eval_steps_per_second": 0.876,
+      "step": 430
+    },
+    {
+      "epoch": 0.536403235843186,
+      "grad_norm": 2.839160680770874,
+      "learning_rate": 4.59640221402214e-06,
+      "loss": 1.6357,
+      "step": 431
+    },
+    {
+      "epoch": 0.537647790914748,
+      "grad_norm": 2.674025535583496,
+      "learning_rate": 4.5940959409594095e-06,
+      "loss": 1.676,
+      "step": 432
+    },
+    {
+      "epoch": 0.5388923459863099,
+      "grad_norm": 3.0619537830352783,
+      "learning_rate": 4.59178966789668e-06,
+      "loss": 1.5617,
+      "step": 433
+    },
+    {
+      "epoch": 0.5401369010578718,
+      "grad_norm": 3.058418035507202,
+      "learning_rate": 4.589483394833948e-06,
+      "loss": 1.5951,
+      "step": 434
+    },
+    {
+      "epoch": 0.5413814561294338,
+      "grad_norm": 3.0046396255493164,
+      "learning_rate": 4.587177121771218e-06,
+      "loss": 1.6473,
+      "step": 435
+    },
+    {
+      "epoch": 0.5426260112009956,
+      "grad_norm": 2.7736752033233643,
+      "learning_rate": 4.584870848708487e-06,
+      "loss": 1.6262,
+      "step": 436
+    },
+    {
+      "epoch": 0.5438705662725576,
+      "grad_norm": 2.7804994583129883,
+      "learning_rate": 4.5825645756457575e-06,
+      "loss": 1.6886,
+      "step": 437
+    },
+    {
+      "epoch": 0.5451151213441194,
+      "grad_norm": 3.0717954635620117,
+      "learning_rate": 4.580258302583026e-06,
+      "loss": 1.6287,
+      "step": 438
+    },
+    {
+      "epoch": 0.5463596764156814,
+      "grad_norm": 2.800243854522705,
+      "learning_rate": 4.5779520295202955e-06,
+      "loss": 1.6169,
+      "step": 439
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "grad_norm": 2.9581644535064697,
+      "learning_rate": 4.575645756457565e-06,
+      "loss": 1.6617,
+      "step": 440
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "eval_loss": 1.7642868757247925,
+      "eval_runtime": 51.2646,
+      "eval_samples_per_second": 19.507,
+      "eval_steps_per_second": 0.819,
+      "step": 440
+    },
+    {
+      "epoch": 0.5488487865588052,
+      "grad_norm": 3.0067460536956787,
+      "learning_rate": 4.573339483394834e-06,
+      "loss": 1.6482,
+      "step": 441
+    },
+    {
+      "epoch": 0.5500933416303672,
+      "grad_norm": 2.9075019359588623,
+      "learning_rate": 4.571033210332104e-06,
+      "loss": 1.6595,
+      "step": 442
+    },
+    {
+      "epoch": 0.551337896701929,
+      "grad_norm": 2.7788755893707275,
+      "learning_rate": 4.568726937269373e-06,
+      "loss": 1.5733,
+      "step": 443
+    },
+    {
+      "epoch": 0.552582451773491,
+      "grad_norm": 2.7775425910949707,
+      "learning_rate": 4.566420664206643e-06,
+      "loss": 1.6925,
+      "step": 444
+    },
+    {
+      "epoch": 0.5538270068450529,
+      "grad_norm": 2.6949567794799805,
+      "learning_rate": 4.564114391143912e-06,
+      "loss": 1.5738,
+      "step": 445
+    },
+    {
+      "epoch": 0.5550715619166148,
+      "grad_norm": 2.7093007564544678,
+      "learning_rate": 4.5618081180811814e-06,
+      "loss": 1.5848,
+      "step": 446
+    },
+    {
+      "epoch": 0.5563161169881767,
+      "grad_norm": 2.6920173168182373,
+      "learning_rate": 4.559501845018451e-06,
+      "loss": 1.6092,
+      "step": 447
+    },
+    {
+      "epoch": 0.5575606720597387,
+      "grad_norm": 2.733731985092163,
+      "learning_rate": 4.5571955719557194e-06,
+      "loss": 1.6056,
+      "step": 448
+    },
+    {
+      "epoch": 0.5588052271313005,
+      "grad_norm": 2.9156274795532227,
+      "learning_rate": 4.55488929889299e-06,
+      "loss": 1.6436,
+      "step": 449
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "grad_norm": 2.8802616596221924,
+      "learning_rate": 4.552583025830259e-06,
+      "loss": 1.6309,
+      "step": 450
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "eval_loss": 1.7486340999603271,
+      "eval_runtime": 47.2219,
+      "eval_samples_per_second": 21.177,
+      "eval_steps_per_second": 0.889,
+      "step": 450
+    },
+    {
+      "epoch": 0.5612943372744243,
+      "grad_norm": 2.729743242263794,
+      "learning_rate": 4.550276752767528e-06,
+      "loss": 1.7171,
+      "step": 451
+    },
+    {
+      "epoch": 0.5625388923459863,
+      "grad_norm": 2.800049066543579,
+      "learning_rate": 4.547970479704797e-06,
+      "loss": 1.6098,
+      "step": 452
+    },
+    {
+      "epoch": 0.5637834474175483,
+      "grad_norm": 2.9659311771392822,
+      "learning_rate": 4.5456642066420666e-06,
+      "loss": 1.6646,
+      "step": 453
+    },
+    {
+      "epoch": 0.5650280024891101,
+      "grad_norm": 2.5417754650115967,
+      "learning_rate": 4.543357933579336e-06,
+      "loss": 1.5847,
+      "step": 454
+    },
+    {
+      "epoch": 0.5662725575606721,
+      "grad_norm": 2.8471338748931885,
+      "learning_rate": 4.541051660516605e-06,
+      "loss": 1.6235,
+      "step": 455
+    },
+    {
+      "epoch": 0.5675171126322339,
+      "grad_norm": 2.7039637565612793,
+      "learning_rate": 4.538745387453875e-06,
+      "loss": 1.5613,
+      "step": 456
+    },
+    {
+      "epoch": 0.5687616677037959,
+      "grad_norm": 3.0383341312408447,
+      "learning_rate": 4.536439114391144e-06,
+      "loss": 1.6485,
+      "step": 457
+    },
+    {
+      "epoch": 0.5700062227753578,
+      "grad_norm": 2.528388261795044,
+      "learning_rate": 4.534132841328414e-06,
+      "loss": 1.6117,
+      "step": 458
+    },
+    {
+      "epoch": 0.5712507778469197,
+      "grad_norm": 2.7623119354248047,
+      "learning_rate": 4.531826568265683e-06,
+      "loss": 1.6017,
+      "step": 459
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "grad_norm": 2.9213945865631104,
+      "learning_rate": 4.5295202952029525e-06,
+      "loss": 1.643,
+      "step": 460
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "eval_loss": 1.747575044631958,
+      "eval_runtime": 45.2844,
+      "eval_samples_per_second": 22.083,
+      "eval_steps_per_second": 0.927,
+      "step": 460
+    },
+    {
+      "epoch": 0.5737398879900436,
+      "grad_norm": 2.664275884628296,
+      "learning_rate": 4.527214022140222e-06,
+      "loss": 1.6154,
+      "step": 461
+    },
+    {
+      "epoch": 0.5749844430616055,
+      "grad_norm": 2.5891788005828857,
+      "learning_rate": 4.524907749077491e-06,
+      "loss": 1.5975,
+      "step": 462
+    },
+    {
+      "epoch": 0.5762289981331674,
+      "grad_norm": 2.8126487731933594,
+      "learning_rate": 4.522601476014761e-06,
+      "loss": 1.6306,
+      "step": 463
+    },
+    {
+      "epoch": 0.5774735532047293,
+      "grad_norm": 2.799391031265259,
+      "learning_rate": 4.520295202952029e-06,
+      "loss": 1.6139,
+      "step": 464
+    },
+    {
+      "epoch": 0.5787181082762912,
+      "grad_norm": 2.7323718070983887,
+      "learning_rate": 4.5179889298893e-06,
+      "loss": 1.632,
+      "step": 465
+    },
+    {
+      "epoch": 0.5799626633478532,
+      "grad_norm": 2.601386785507202,
+      "learning_rate": 4.515682656826569e-06,
+      "loss": 1.6139,
+      "step": 466
+    },
+    {
+      "epoch": 0.581207218419415,
+      "grad_norm": 2.562162399291992,
+      "learning_rate": 4.513376383763838e-06,
+      "loss": 1.5168,
+      "step": 467
+    },
+    {
+      "epoch": 0.582451773490977,
+      "grad_norm": 2.6912002563476562,
+      "learning_rate": 4.511070110701107e-06,
+      "loss": 1.6249,
+      "step": 468
+    },
+    {
+      "epoch": 0.583696328562539,
+      "grad_norm": 2.6971495151519775,
+      "learning_rate": 4.5087638376383765e-06,
+      "loss": 1.616,
+      "step": 469
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "grad_norm": 2.6898910999298096,
+      "learning_rate": 4.506457564575646e-06,
+      "loss": 1.614,
+      "step": 470
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "eval_loss": 1.7460769414901733,
+      "eval_runtime": 45.3055,
+      "eval_samples_per_second": 22.072,
+      "eval_steps_per_second": 0.927,
+      "step": 470
+    },
+    {
+      "epoch": 0.5861854387056628,
+      "grad_norm": 2.698180675506592,
+      "learning_rate": 4.504151291512915e-06,
+      "loss": 1.5993,
+      "step": 471
+    },
+    {
+      "epoch": 0.5874299937772246,
+      "grad_norm": 2.7125210762023926,
+      "learning_rate": 4.501845018450185e-06,
+      "loss": 1.6113,
+      "step": 472
+    },
+    {
+      "epoch": 0.5886745488487866,
+      "grad_norm": 2.754445791244507,
+      "learning_rate": 4.499538745387454e-06,
+      "loss": 1.6719,
+      "step": 473
+    },
+    {
+      "epoch": 0.5899191039203485,
+      "grad_norm": 2.6979362964630127,
+      "learning_rate": 4.497232472324724e-06,
+      "loss": 1.608,
+      "step": 474
+    },
+    {
+      "epoch": 0.5911636589919104,
+      "grad_norm": 2.7694528102874756,
+      "learning_rate": 4.494926199261993e-06,
+      "loss": 1.5695,
+      "step": 475
+    },
+    {
+      "epoch": 0.5924082140634723,
+      "grad_norm": 2.653353214263916,
+      "learning_rate": 4.4926199261992624e-06,
+      "loss": 1.5523,
+      "step": 476
+    },
+    {
+      "epoch": 0.5936527691350342,
+      "grad_norm": 2.6632070541381836,
+      "learning_rate": 4.490313653136532e-06,
+      "loss": 1.5675,
+      "step": 477
+    },
+    {
+      "epoch": 0.5948973242065961,
+      "grad_norm": 3.038543939590454,
+      "learning_rate": 4.488007380073801e-06,
+      "loss": 1.5692,
+      "step": 478
+    },
+    {
+      "epoch": 0.5961418792781581,
+      "grad_norm": 2.8123953342437744,
+      "learning_rate": 4.485701107011071e-06,
+      "loss": 1.6223,
+      "step": 479
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "grad_norm": 2.5845773220062256,
+      "learning_rate": 4.483394833948339e-06,
+      "loss": 1.5773,
+      "step": 480
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "eval_loss": 1.7419319152832031,
+      "eval_runtime": 44.6842,
+      "eval_samples_per_second": 22.379,
+      "eval_steps_per_second": 0.94,
+      "step": 480
+    },
+    {
+      "epoch": 0.5986309894212819,
+      "grad_norm": 3.069936752319336,
+      "learning_rate": 4.4810885608856096e-06,
+      "loss": 1.6323,
+      "step": 481
+    },
+    {
+      "epoch": 0.5998755444928439,
+      "grad_norm": 2.9085781574249268,
+      "learning_rate": 4.478782287822879e-06,
+      "loss": 1.5845,
+      "step": 482
+    },
+    {
+      "epoch": 0.6011200995644057,
+      "grad_norm": 2.8568010330200195,
+      "learning_rate": 4.4764760147601476e-06,
+      "loss": 1.5898,
+      "step": 483
+    },
+    {
+      "epoch": 0.6023646546359677,
+      "grad_norm": 3.089081048965454,
+      "learning_rate": 4.474169741697417e-06,
+      "loss": 1.6622,
+      "step": 484
+    },
+    {
+      "epoch": 0.6036092097075295,
+      "grad_norm": 2.8515470027923584,
+      "learning_rate": 4.471863468634686e-06,
+      "loss": 1.5859,
+      "step": 485
+    },
+    {
+      "epoch": 0.6048537647790915,
+      "grad_norm": 2.647765874862671,
+      "learning_rate": 4.469557195571957e-06,
+      "loss": 1.5926,
+      "step": 486
+    },
+    {
+      "epoch": 0.6060983198506534,
+      "grad_norm": 2.61676025390625,
+      "learning_rate": 4.467250922509225e-06,
+      "loss": 1.5684,
+      "step": 487
+    },
+    {
+      "epoch": 0.6073428749222153,
+      "grad_norm": 2.7127039432525635,
+      "learning_rate": 4.464944649446495e-06,
+      "loss": 1.5702,
+      "step": 488
+    },
+    {
+      "epoch": 0.6085874299937772,
+      "grad_norm": 2.9932055473327637,
+      "learning_rate": 4.462638376383764e-06,
+      "loss": 1.5976,
+      "step": 489
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "grad_norm": 2.6508774757385254,
+      "learning_rate": 4.4603321033210335e-06,
+      "loss": 1.586,
+      "step": 490
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "eval_loss": 1.7357326745986938,
+      "eval_runtime": 45.393,
+      "eval_samples_per_second": 22.03,
+      "eval_steps_per_second": 0.925,
+      "step": 490
+    },
+    {
+      "epoch": 0.611076540136901,
+      "grad_norm": 2.7626044750213623,
+      "learning_rate": 4.458025830258303e-06,
+      "loss": 1.5773,
+      "step": 491
+    },
+    {
+      "epoch": 0.612321095208463,
+      "grad_norm": 2.595003604888916,
+      "learning_rate": 4.455719557195572e-06,
+      "loss": 1.601,
+      "step": 492
+    },
+    {
+      "epoch": 0.6135656502800249,
+      "grad_norm": 2.595767021179199,
+      "learning_rate": 4.453413284132842e-06,
+      "loss": 1.6287,
+      "step": 493
+    },
+    {
+      "epoch": 0.6148102053515868,
+      "grad_norm": 2.755845308303833,
+      "learning_rate": 4.451107011070111e-06,
+      "loss": 1.593,
+      "step": 494
+    },
+    {
+      "epoch": 0.6160547604231488,
+      "grad_norm": 2.7302653789520264,
+      "learning_rate": 4.448800738007381e-06,
+      "loss": 1.5768,
+      "step": 495
+    },
+    {
+      "epoch": 0.6172993154947106,
+      "grad_norm": 2.6493024826049805,
+      "learning_rate": 4.446494464944649e-06,
+      "loss": 1.6059,
+      "step": 496
+    },
+    {
+      "epoch": 0.6185438705662726,
+      "grad_norm": 2.8633735179901123,
+      "learning_rate": 4.4441881918819195e-06,
+      "loss": 1.598,
+      "step": 497
+    },
+    {
+      "epoch": 0.6197884256378344,
+      "grad_norm": 2.65639066696167,
+      "learning_rate": 4.441881918819189e-06,
+      "loss": 1.6036,
+      "step": 498
+    },
+    {
+      "epoch": 0.6210329807093964,
+      "grad_norm": 2.6867759227752686,
+      "learning_rate": 4.439575645756458e-06,
+      "loss": 1.5676,
+      "step": 499
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "grad_norm": 2.7986443042755127,
+      "learning_rate": 4.437269372693727e-06,
+      "loss": 1.524,
+      "step": 500
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "eval_loss": 1.7449125051498413,
+      "eval_runtime": 43.9329,
+      "eval_samples_per_second": 22.762,
+      "eval_steps_per_second": 0.956,
+      "step": 500
+    },
+    {
+      "epoch": 0.6235220908525202,
+      "grad_norm": 2.665905714035034,
+      "learning_rate": 4.434963099630996e-06,
+      "loss": 1.5635,
+      "step": 501
+    },
+    {
+      "epoch": 0.6247666459240822,
+      "grad_norm": 2.902435541152954,
+      "learning_rate": 4.432656826568267e-06,
+      "loss": 1.6377,
+      "step": 502
+    },
+    {
+      "epoch": 0.6260112009956441,
+      "grad_norm": 2.6872262954711914,
+      "learning_rate": 4.430350553505535e-06,
+      "loss": 1.5553,
+      "step": 503
+    },
+    {
+      "epoch": 0.627255756067206,
+      "grad_norm": 2.676621913909912,
+      "learning_rate": 4.428044280442805e-06,
+      "loss": 1.6101,
+      "step": 504
+    },
+    {
+      "epoch": 0.6285003111387679,
+      "grad_norm": 2.756347179412842,
+      "learning_rate": 4.425738007380074e-06,
+      "loss": 1.5884,
+      "step": 505
+    },
+    {
+      "epoch": 0.6297448662103298,
+      "grad_norm": 2.856882333755493,
+      "learning_rate": 4.4234317343173434e-06,
+      "loss": 1.5635,
+      "step": 506
+    },
+    {
+      "epoch": 0.6309894212818917,
+      "grad_norm": 2.7708330154418945,
+      "learning_rate": 4.421125461254613e-06,
+      "loss": 1.6028,
+      "step": 507
+    },
+    {
+      "epoch": 0.6322339763534537,
+      "grad_norm": 2.8167600631713867,
+      "learning_rate": 4.418819188191882e-06,
+      "loss": 1.6026,
+      "step": 508
+    },
+    {
+      "epoch": 0.6334785314250155,
+      "grad_norm": 2.54194974899292,
+      "learning_rate": 4.416512915129152e-06,
+      "loss": 1.6001,
+      "step": 509
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "grad_norm": 2.683037519454956,
+      "learning_rate": 4.414206642066421e-06,
+      "loss": 1.6084,
+      "step": 510
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "eval_loss": 1.7376186847686768,
+      "eval_runtime": 42.3026,
+      "eval_samples_per_second": 23.639,
+      "eval_steps_per_second": 0.993,
+      "step": 510
+    },
+    {
+      "epoch": 0.6359676415681393,
+      "grad_norm": 2.82627010345459,
+      "learning_rate": 4.4119003690036905e-06,
+      "loss": 1.6258,
+      "step": 511
+    },
+    {
+      "epoch": 0.6372121966397013,
+      "grad_norm": 2.6139848232269287,
+      "learning_rate": 4.40959409594096e-06,
+      "loss": 1.6029,
+      "step": 512
+    },
+    {
+      "epoch": 0.6384567517112633,
+      "grad_norm": 2.646712303161621,
+      "learning_rate": 4.407287822878229e-06,
+      "loss": 1.5602,
+      "step": 513
+    },
+    {
+      "epoch": 0.6397013067828251,
+      "grad_norm": 2.910935163497925,
+      "learning_rate": 4.404981549815499e-06,
+      "loss": 1.5757,
+      "step": 514
+    },
+    {
+      "epoch": 0.6409458618543871,
+      "grad_norm": 2.6001148223876953,
+      "learning_rate": 4.402675276752768e-06,
+      "loss": 1.5857,
+      "step": 515
+    },
+    {
+      "epoch": 0.642190416925949,
+      "grad_norm": 2.8525235652923584,
+      "learning_rate": 4.400369003690037e-06,
+      "loss": 1.6092,
+      "step": 516
+    },
+    {
+      "epoch": 0.6434349719975109,
+      "grad_norm": 2.5335488319396973,
+      "learning_rate": 4.398062730627306e-06,
+      "loss": 1.5849,
+      "step": 517
+    },
+    {
+      "epoch": 0.6446795270690728,
+      "grad_norm": 2.5787103176116943,
+      "learning_rate": 4.3957564575645765e-06,
+      "loss": 1.634,
+      "step": 518
+    },
+    {
+      "epoch": 0.6459240821406347,
+      "grad_norm": 2.6188197135925293,
+      "learning_rate": 4.393450184501845e-06,
+      "loss": 1.5477,
+      "step": 519
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "grad_norm": 2.6548666954040527,
+      "learning_rate": 4.3911439114391145e-06,
+      "loss": 1.6133,
+      "step": 520
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "eval_loss": 1.7393039464950562,
+      "eval_runtime": 45.9234,
+      "eval_samples_per_second": 21.775,
+      "eval_steps_per_second": 0.915,
+      "step": 520
+    },
+    {
+      "epoch": 0.6484131922837586,
+      "grad_norm": 2.907928228378296,
+      "learning_rate": 4.388837638376384e-06,
+      "loss": 1.587,
+      "step": 521
+    },
+    {
+      "epoch": 0.6496577473553204,
+      "grad_norm": 2.7899692058563232,
+      "learning_rate": 4.386531365313653e-06,
+      "loss": 1.6126,
+      "step": 522
+    },
+    {
+      "epoch": 0.6509023024268824,
+      "grad_norm": 2.680147886276245,
+      "learning_rate": 4.384225092250923e-06,
+      "loss": 1.565,
+      "step": 523
+    },
+    {
+      "epoch": 0.6521468574984443,
+      "grad_norm": 2.6590754985809326,
+      "learning_rate": 4.381918819188192e-06,
+      "loss": 1.6127,
+      "step": 524
+    },
+    {
+      "epoch": 0.6533914125700062,
+      "grad_norm": 2.6638906002044678,
+      "learning_rate": 4.379612546125462e-06,
+      "loss": 1.5615,
+      "step": 525
+    },
+    {
+      "epoch": 0.6546359676415682,
+      "grad_norm": 2.6518194675445557,
+      "learning_rate": 4.377306273062731e-06,
+      "loss": 1.6202,
+      "step": 526
+    },
+    {
+      "epoch": 0.65588052271313,
+      "grad_norm": 2.5323619842529297,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 1.5305,
+      "step": 527
+    },
+    {
+      "epoch": 0.657125077784692,
+      "grad_norm": 2.6871442794799805,
+      "learning_rate": 4.37269372693727e-06,
+      "loss": 1.6043,
+      "step": 528
+    },
+    {
+      "epoch": 0.6583696328562539,
+      "grad_norm": 2.760746717453003,
+      "learning_rate": 4.370387453874539e-06,
+      "loss": 1.6059,
+      "step": 529
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "grad_norm": 2.780482769012451,
+      "learning_rate": 4.368081180811809e-06,
+      "loss": 1.6082,
+      "step": 530
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "eval_loss": 1.7219713926315308,
+      "eval_runtime": 51.8478,
+      "eval_samples_per_second": 19.287,
+      "eval_steps_per_second": 0.81,
+      "step": 530
+    },
+    {
+      "epoch": 0.6608587429993777,
+      "grad_norm": 2.605890989303589,
+      "learning_rate": 4.365774907749078e-06,
+      "loss": 1.5554,
+      "step": 531
+    },
+    {
+      "epoch": 0.6621032980709396,
+      "grad_norm": 2.731555461883545,
+      "learning_rate": 4.363468634686347e-06,
+      "loss": 1.606,
+      "step": 532
+    },
+    {
+      "epoch": 0.6633478531425016,
+      "grad_norm": 2.8356943130493164,
+      "learning_rate": 4.361162361623616e-06,
+      "loss": 1.5601,
+      "step": 533
+    },
+    {
+      "epoch": 0.6645924082140635,
+      "grad_norm": 2.7196593284606934,
+      "learning_rate": 4.3588560885608864e-06,
+      "loss": 1.5722,
+      "step": 534
+    },
+    {
+      "epoch": 0.6658369632856254,
+      "grad_norm": 2.621371269226074,
+      "learning_rate": 4.356549815498156e-06,
+      "loss": 1.6194,
+      "step": 535
+    },
+    {
+      "epoch": 0.6670815183571873,
+      "grad_norm": 2.653916120529175,
+      "learning_rate": 4.354243542435424e-06,
+      "loss": 1.5592,
+      "step": 536
+    },
+    {
+      "epoch": 0.6683260734287492,
+      "grad_norm": 2.88431453704834,
+      "learning_rate": 4.351937269372694e-06,
+      "loss": 1.6296,
+      "step": 537
+    },
+    {
+      "epoch": 0.6695706285003111,
+      "grad_norm": 2.667130470275879,
+      "learning_rate": 4.349630996309963e-06,
+      "loss": 1.5624,
+      "step": 538
+    },
+    {
+      "epoch": 0.6708151835718731,
+      "grad_norm": 2.6453566551208496,
+      "learning_rate": 4.347324723247233e-06,
+      "loss": 1.6025,
+      "step": 539
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "grad_norm": 3.032271146774292,
+      "learning_rate": 4.345018450184502e-06,
+      "loss": 1.6771,
+      "step": 540
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "eval_loss": 1.721895456314087,
+      "eval_runtime": 50.62,
+      "eval_samples_per_second": 19.755,
+      "eval_steps_per_second": 0.83,
+      "step": 540
+    },
+    {
+      "epoch": 0.6733042937149969,
+      "grad_norm": 2.8476362228393555,
+      "learning_rate": 4.3427121771217715e-06,
+      "loss": 1.5663,
+      "step": 541
+    },
+    {
+      "epoch": 0.6745488487865589,
+      "grad_norm": 2.615602970123291,
+      "learning_rate": 4.340405904059041e-06,
+      "loss": 1.5815,
+      "step": 542
+    },
+    {
+      "epoch": 0.6757934038581207,
+      "grad_norm": 3.221571683883667,
+      "learning_rate": 4.33809963099631e-06,
+      "loss": 1.5955,
+      "step": 543
+    },
+    {
+      "epoch": 0.6770379589296827,
+      "grad_norm": 2.602994203567505,
+      "learning_rate": 4.33579335793358e-06,
+      "loss": 1.5667,
+      "step": 544
+    },
+    {
+      "epoch": 0.6782825140012445,
+      "grad_norm": 2.9380433559417725,
+      "learning_rate": 4.333487084870848e-06,
+      "loss": 1.5651,
+      "step": 545
+    },
+    {
+      "epoch": 0.6795270690728065,
+      "grad_norm": 3.0689425468444824,
+      "learning_rate": 4.331180811808119e-06,
+      "loss": 1.6003,
+      "step": 546
+    },
+    {
+      "epoch": 0.6807716241443684,
+      "grad_norm": 2.770317316055298,
+      "learning_rate": 4.328874538745388e-06,
+      "loss": 1.5576,
+      "step": 547
+    },
+    {
+      "epoch": 0.6820161792159303,
+      "grad_norm": 2.8631815910339355,
+      "learning_rate": 4.3265682656826575e-06,
+      "loss": 1.5308,
+      "step": 548
+    },
+    {
+      "epoch": 0.6832607342874922,
+      "grad_norm": 2.785576820373535,
+      "learning_rate": 4.324261992619926e-06,
+      "loss": 1.6113,
+      "step": 549
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "grad_norm": 2.723919630050659,
+      "learning_rate": 4.321955719557196e-06,
+      "loss": 1.5606,
+      "step": 550
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "eval_loss": 1.721803069114685,
+      "eval_runtime": 51.6036,
+      "eval_samples_per_second": 19.378,
+      "eval_steps_per_second": 0.814,
+      "step": 550
+    },
+    {
+      "epoch": 0.685749844430616,
+      "grad_norm": 3.081198215484619,
+      "learning_rate": 4.319649446494466e-06,
+      "loss": 1.5643,
+      "step": 551
+    },
+    {
+      "epoch": 0.686994399502178,
+      "grad_norm": 2.757392168045044,
+      "learning_rate": 4.317343173431734e-06,
+      "loss": 1.5909,
+      "step": 552
+    },
+    {
+      "epoch": 0.6882389545737398,
+      "grad_norm": 3.038753032684326,
+      "learning_rate": 4.315036900369004e-06,
+      "loss": 1.6023,
+      "step": 553
+    },
+    {
+      "epoch": 0.6894835096453018,
+      "grad_norm": 2.8246452808380127,
+      "learning_rate": 4.312730627306273e-06,
+      "loss": 1.5548,
+      "step": 554
+    },
+    {
+      "epoch": 0.6907280647168638,
+      "grad_norm": 2.589320182800293,
+      "learning_rate": 4.310424354243543e-06,
+      "loss": 1.5424,
+      "step": 555
+    },
+    {
+      "epoch": 0.6919726197884256,
+      "grad_norm": 2.6413373947143555,
+      "learning_rate": 4.308118081180812e-06,
+      "loss": 1.6243,
+      "step": 556
+    },
+    {
+      "epoch": 0.6932171748599876,
+      "grad_norm": 2.764784336090088,
+      "learning_rate": 4.3058118081180815e-06,
+      "loss": 1.589,
+      "step": 557
+    },
+    {
+      "epoch": 0.6944617299315494,
+      "grad_norm": 2.806821823120117,
+      "learning_rate": 4.303505535055351e-06,
+      "loss": 1.621,
+      "step": 558
+    },
+    {
+      "epoch": 0.6957062850031114,
+      "grad_norm": 2.718017578125,
+      "learning_rate": 4.30119926199262e-06,
+      "loss": 1.5773,
+      "step": 559
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "grad_norm": 2.7620160579681396,
+      "learning_rate": 4.29889298892989e-06,
+      "loss": 1.5868,
+      "step": 560
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "eval_loss": 1.7152249813079834,
+      "eval_runtime": 52.0463,
+      "eval_samples_per_second": 19.214,
+      "eval_steps_per_second": 0.807,
+      "step": 560
+    },
+    {
+      "epoch": 0.6981953951462352,
+      "grad_norm": 2.66686749458313,
+      "learning_rate": 4.296586715867159e-06,
+      "loss": 1.534,
+      "step": 561
+    },
+    {
+      "epoch": 0.6994399502177971,
+      "grad_norm": 2.812577247619629,
+      "learning_rate": 4.2942804428044286e-06,
+      "loss": 1.5501,
+      "step": 562
+    },
+    {
+      "epoch": 0.7006845052893591,
+      "grad_norm": 2.578508138656616,
+      "learning_rate": 4.291974169741698e-06,
+      "loss": 1.6286,
+      "step": 563
+    },
+    {
+      "epoch": 0.701929060360921,
+      "grad_norm": 2.524249315261841,
+      "learning_rate": 4.289667896678967e-06,
+      "loss": 1.5719,
+      "step": 564
+    },
+    {
+      "epoch": 0.7031736154324829,
+      "grad_norm": 2.827235460281372,
+      "learning_rate": 4.287361623616236e-06,
+      "loss": 1.5333,
+      "step": 565
+    },
+    {
+      "epoch": 0.7044181705040448,
+      "grad_norm": 2.6359963417053223,
+      "learning_rate": 4.285055350553506e-06,
+      "loss": 1.5849,
+      "step": 566
+    },
+    {
+      "epoch": 0.7056627255756067,
+      "grad_norm": 2.930530071258545,
+      "learning_rate": 4.282749077490776e-06,
+      "loss": 1.5672,
+      "step": 567
+    },
+    {
+      "epoch": 0.7069072806471687,
+      "grad_norm": 2.750102996826172,
+      "learning_rate": 4.280442804428044e-06,
+      "loss": 1.5696,
+      "step": 568
+    },
+    {
+      "epoch": 0.7081518357187305,
+      "grad_norm": 2.869690418243408,
+      "learning_rate": 4.278136531365314e-06,
+      "loss": 1.5689,
+      "step": 569
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "grad_norm": 2.954852819442749,
+      "learning_rate": 4.275830258302583e-06,
+      "loss": 1.5931,
+      "step": 570
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "eval_loss": 1.7143300771713257,
+      "eval_runtime": 52.7293,
+      "eval_samples_per_second": 18.965,
+      "eval_steps_per_second": 0.797,
+      "step": 570
+    },
+    {
+      "epoch": 0.7106409458618543,
+      "grad_norm": 2.702223539352417,
+      "learning_rate": 4.273523985239853e-06,
+      "loss": 1.5635,
+      "step": 571
+    },
+    {
+      "epoch": 0.7118855009334163,
+      "grad_norm": 2.689995050430298,
+      "learning_rate": 4.271217712177122e-06,
+      "loss": 1.5545,
+      "step": 572
+    },
+    {
+      "epoch": 0.7131300560049783,
+      "grad_norm": 2.68979811668396,
+      "learning_rate": 4.268911439114391e-06,
+      "loss": 1.5404,
+      "step": 573
+    },
+    {
+      "epoch": 0.7143746110765401,
+      "grad_norm": 2.7477986812591553,
+      "learning_rate": 4.266605166051661e-06,
+      "loss": 1.5719,
+      "step": 574
+    },
+    {
+      "epoch": 0.7156191661481021,
+      "grad_norm": 2.975778341293335,
+      "learning_rate": 4.26429889298893e-06,
+      "loss": 1.5587,
+      "step": 575
+    },
+    {
+      "epoch": 0.716863721219664,
+      "grad_norm": 2.658170700073242,
+      "learning_rate": 4.2619926199262e-06,
+      "loss": 1.5746,
+      "step": 576
+    },
+    {
+      "epoch": 0.7181082762912259,
+      "grad_norm": 2.8825011253356934,
+      "learning_rate": 4.259686346863469e-06,
+      "loss": 1.5246,
+      "step": 577
+    },
+    {
+      "epoch": 0.7193528313627878,
+      "grad_norm": 2.845280170440674,
+      "learning_rate": 4.2573800738007385e-06,
+      "loss": 1.5703,
+      "step": 578
+    },
+    {
+      "epoch": 0.7205973864343497,
+      "grad_norm": 2.660616636276245,
+      "learning_rate": 4.255073800738008e-06,
+      "loss": 1.5373,
+      "step": 579
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "grad_norm": 2.749447822570801,
+      "learning_rate": 4.252767527675277e-06,
+      "loss": 1.59,
+      "step": 580
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "eval_loss": 1.7149444818496704,
+      "eval_runtime": 52.2099,
+      "eval_samples_per_second": 19.153,
+      "eval_steps_per_second": 0.804,
+      "step": 580
+    },
+    {
+      "epoch": 0.7230864965774736,
+      "grad_norm": 2.813328742980957,
+      "learning_rate": 4.250461254612546e-06,
+      "loss": 1.5694,
+      "step": 581
+    },
+    {
+      "epoch": 0.7243310516490354,
+      "grad_norm": 2.823866844177246,
+      "learning_rate": 4.248154981549816e-06,
+      "loss": 1.5143,
+      "step": 582
+    },
+    {
+      "epoch": 0.7255756067205974,
+      "grad_norm": 2.71337890625,
+      "learning_rate": 4.245848708487086e-06,
+      "loss": 1.5915,
+      "step": 583
+    },
+    {
+      "epoch": 0.7268201617921594,
+      "grad_norm": 2.718085765838623,
+      "learning_rate": 4.243542435424355e-06,
+      "loss": 1.604,
+      "step": 584
+    },
+    {
+      "epoch": 0.7280647168637212,
+      "grad_norm": 2.5741796493530273,
+      "learning_rate": 4.241236162361624e-06,
+      "loss": 1.5398,
+      "step": 585
+    },
+    {
+      "epoch": 0.7293092719352832,
+      "grad_norm": 2.727114200592041,
+      "learning_rate": 4.238929889298893e-06,
+      "loss": 1.5886,
+      "step": 586
+    },
+    {
+      "epoch": 0.730553827006845,
+      "grad_norm": 2.6703338623046875,
+      "learning_rate": 4.236623616236163e-06,
+      "loss": 1.5959,
+      "step": 587
+    },
+    {
+      "epoch": 0.731798382078407,
+      "grad_norm": 2.750326633453369,
+      "learning_rate": 4.234317343173432e-06,
+      "loss": 1.5916,
+      "step": 588
+    },
+    {
+      "epoch": 0.7330429371499689,
+      "grad_norm": 2.6680715084075928,
+      "learning_rate": 4.232011070110701e-06,
+      "loss": 1.562,
+      "step": 589
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "grad_norm": 2.6603832244873047,
+      "learning_rate": 4.229704797047971e-06,
+      "loss": 1.6121,
+      "step": 590
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "eval_loss": 1.7144734859466553,
+      "eval_runtime": 50.5317,
+      "eval_samples_per_second": 19.79,
+      "eval_steps_per_second": 0.831,
+      "step": 590
+    },
+    {
+      "epoch": 0.7355320472930927,
+      "grad_norm": 2.7868523597717285,
+      "learning_rate": 4.22739852398524e-06,
+      "loss": 1.5488,
+      "step": 591
+    },
+    {
+      "epoch": 0.7367766023646546,
+      "grad_norm": 2.755270004272461,
+      "learning_rate": 4.2250922509225096e-06,
+      "loss": 1.5801,
+      "step": 592
+    },
+    {
+      "epoch": 0.7380211574362165,
+      "grad_norm": 2.7614126205444336,
+      "learning_rate": 4.222785977859779e-06,
+      "loss": 1.5507,
+      "step": 593
+    },
+    {
+      "epoch": 0.7392657125077785,
+      "grad_norm": 2.8674862384796143,
+      "learning_rate": 4.220479704797048e-06,
+      "loss": 1.5807,
+      "step": 594
+    },
+    {
+      "epoch": 0.7405102675793404,
+      "grad_norm": 2.7366859912872314,
+      "learning_rate": 4.218173431734318e-06,
+      "loss": 1.6594,
+      "step": 595
+    },
+    {
+      "epoch": 0.7417548226509023,
+      "grad_norm": 2.6231045722961426,
+      "learning_rate": 4.215867158671587e-06,
+      "loss": 1.5609,
+      "step": 596
+    },
+    {
+      "epoch": 0.7429993777224643,
+      "grad_norm": 2.7694146633148193,
+      "learning_rate": 4.213560885608857e-06,
+      "loss": 1.5889,
+      "step": 597
+    },
+    {
+      "epoch": 0.7442439327940261,
+      "grad_norm": 2.6275386810302734,
+      "learning_rate": 4.211254612546125e-06,
+      "loss": 1.575,
+      "step": 598
+    },
+    {
+      "epoch": 0.7454884878655881,
+      "grad_norm": 2.9413866996765137,
+      "learning_rate": 4.2089483394833955e-06,
+      "loss": 1.589,
+      "step": 599
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "grad_norm": 2.768606185913086,
+      "learning_rate": 4.206642066420665e-06,
+      "loss": 1.5738,
+      "step": 600
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "eval_loss": 1.7148027420043945,
+      "eval_runtime": 54.7031,
+      "eval_samples_per_second": 18.28,
+      "eval_steps_per_second": 0.768,
+      "step": 600
+    },
+    {
+      "epoch": 0.7479775980087119,
+      "grad_norm": 2.7917706966400146,
+      "learning_rate": 4.2043357933579335e-06,
+      "loss": 1.499,
+      "step": 601
+    },
+    {
+      "epoch": 0.7492221530802738,
+      "grad_norm": 2.5982208251953125,
+      "learning_rate": 4.202029520295203e-06,
+      "loss": 1.5113,
+      "step": 602
+    },
+    {
+      "epoch": 0.7504667081518357,
+      "grad_norm": 2.692032814025879,
+      "learning_rate": 4.199723247232473e-06,
+      "loss": 1.4721,
+      "step": 603
+    },
+    {
+      "epoch": 0.7517112632233977,
+      "grad_norm": 2.975860834121704,
+      "learning_rate": 4.197416974169742e-06,
+      "loss": 1.5147,
+      "step": 604
+    },
+    {
+      "epoch": 0.7529558182949595,
+      "grad_norm": 2.758610248565674,
+      "learning_rate": 4.195110701107011e-06,
+      "loss": 1.5674,
+      "step": 605
+    },
+    {
+      "epoch": 0.7542003733665215,
+      "grad_norm": 2.7620184421539307,
+      "learning_rate": 4.192804428044281e-06,
+      "loss": 1.5669,
+      "step": 606
+    },
+    {
+      "epoch": 0.7554449284380834,
+      "grad_norm": 3.043940305709839,
+      "learning_rate": 4.19049815498155e-06,
+      "loss": 1.587,
+      "step": 607
+    },
+    {
+      "epoch": 0.7566894835096453,
+      "grad_norm": 2.7482151985168457,
+      "learning_rate": 4.1881918819188195e-06,
+      "loss": 1.5451,
+      "step": 608
+    },
+    {
+      "epoch": 0.7579340385812072,
+      "grad_norm": 2.7259294986724854,
+      "learning_rate": 4.185885608856089e-06,
+      "loss": 1.5295,
+      "step": 609
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "grad_norm": 2.6340832710266113,
+      "learning_rate": 4.183579335793358e-06,
+      "loss": 1.5335,
+      "step": 610
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "eval_loss": 1.7038393020629883,
+      "eval_runtime": 44.0792,
+      "eval_samples_per_second": 22.686,
+      "eval_steps_per_second": 0.953,
+      "step": 610
+    },
+    {
+      "epoch": 0.760423148724331,
+      "grad_norm": 2.6144909858703613,
+      "learning_rate": 4.181273062730628e-06,
+      "loss": 1.5907,
+      "step": 611
+    },
+    {
+      "epoch": 0.761667703795893,
+      "grad_norm": 2.7754175662994385,
+      "learning_rate": 4.178966789667897e-06,
+      "loss": 1.5773,
+      "step": 612
+    },
+    {
+      "epoch": 0.7629122588674548,
+      "grad_norm": 2.6313252449035645,
+      "learning_rate": 4.176660516605167e-06,
+      "loss": 1.5675,
+      "step": 613
+    },
+    {
+      "epoch": 0.7641568139390168,
+      "grad_norm": 2.549074649810791,
+      "learning_rate": 4.174354243542435e-06,
+      "loss": 1.595,
+      "step": 614
+    },
+    {
+      "epoch": 0.7654013690105788,
+      "grad_norm": 2.611804246902466,
+      "learning_rate": 4.1720479704797054e-06,
+      "loss": 1.5416,
+      "step": 615
+    },
+    {
+      "epoch": 0.7666459240821406,
+      "grad_norm": 2.6322927474975586,
+      "learning_rate": 4.169741697416975e-06,
+      "loss": 1.604,
+      "step": 616
+    },
+    {
+      "epoch": 0.7678904791537026,
+      "grad_norm": 2.5792219638824463,
+      "learning_rate": 4.1674354243542434e-06,
+      "loss": 1.5976,
+      "step": 617
+    },
+    {
+      "epoch": 0.7691350342252644,
+      "grad_norm": 2.845416307449341,
+      "learning_rate": 4.165129151291513e-06,
+      "loss": 1.5945,
+      "step": 618
+    },
+    {
+      "epoch": 0.7703795892968264,
+      "grad_norm": 2.8647871017456055,
+      "learning_rate": 4.162822878228783e-06,
+      "loss": 1.5637,
+      "step": 619
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "grad_norm": 2.590719699859619,
+      "learning_rate": 4.1605166051660526e-06,
+      "loss": 1.556,
+      "step": 620
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "eval_loss": 1.7115222215652466,
+      "eval_runtime": 50.3905,
+      "eval_samples_per_second": 19.845,
+      "eval_steps_per_second": 0.833,
+      "step": 620
+    },
+    {
+      "epoch": 0.7728686994399502,
+      "grad_norm": 2.626723527908325,
+      "learning_rate": 4.158210332103321e-06,
+      "loss": 1.5486,
+      "step": 621
+    },
+    {
+      "epoch": 0.7741132545115121,
+      "grad_norm": 2.713069438934326,
+      "learning_rate": 4.1559040590405906e-06,
+      "loss": 1.5474,
+      "step": 622
+    },
+    {
+      "epoch": 0.7753578095830741,
+      "grad_norm": 2.685563087463379,
+      "learning_rate": 4.15359778597786e-06,
+      "loss": 1.5529,
+      "step": 623
+    },
+    {
+      "epoch": 0.776602364654636,
+      "grad_norm": 2.690586805343628,
+      "learning_rate": 4.151291512915129e-06,
+      "loss": 1.5293,
+      "step": 624
+    },
+    {
+      "epoch": 0.7778469197261979,
+      "grad_norm": 2.865345001220703,
+      "learning_rate": 4.148985239852399e-06,
+      "loss": 1.5095,
+      "step": 625
+    },
+    {
+      "epoch": 0.7790914747977598,
+      "grad_norm": 2.92393159866333,
+      "learning_rate": 4.146678966789668e-06,
+      "loss": 1.5561,
+      "step": 626
+    },
+    {
+      "epoch": 0.7803360298693217,
+      "grad_norm": 2.554480791091919,
+      "learning_rate": 4.144372693726938e-06,
+      "loss": 1.4876,
+      "step": 627
+    },
+    {
+      "epoch": 0.7815805849408837,
+      "grad_norm": 2.6128833293914795,
+      "learning_rate": 4.142066420664207e-06,
+      "loss": 1.5378,
+      "step": 628
+    },
+    {
+      "epoch": 0.7828251400124455,
+      "grad_norm": 3.0110888481140137,
+      "learning_rate": 4.1397601476014765e-06,
+      "loss": 1.5859,
+      "step": 629
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "grad_norm": 2.8152706623077393,
+      "learning_rate": 4.137453874538745e-06,
+      "loss": 1.5841,
+      "step": 630
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "eval_loss": 1.702181339263916,
+      "eval_runtime": 53.6926,
+      "eval_samples_per_second": 18.625,
+      "eval_steps_per_second": 0.782,
+      "step": 630
+    },
+    {
+      "epoch": 0.7853142501555694,
+      "grad_norm": 2.8634233474731445,
+      "learning_rate": 4.135147601476015e-06,
+      "loss": 1.564,
+      "step": 631
+    },
+    {
+      "epoch": 0.7865588052271313,
+      "grad_norm": 2.8939666748046875,
+      "learning_rate": 4.132841328413285e-06,
+      "loss": 1.5859,
+      "step": 632
+    },
+    {
+      "epoch": 0.7878033602986932,
+      "grad_norm": 2.6560444831848145,
+      "learning_rate": 4.130535055350554e-06,
+      "loss": 1.5199,
+      "step": 633
+    },
+    {
+      "epoch": 0.7890479153702551,
+      "grad_norm": 2.964721202850342,
+      "learning_rate": 4.128228782287823e-06,
+      "loss": 1.5888,
+      "step": 634
+    },
+    {
+      "epoch": 0.790292470441817,
+      "grad_norm": 2.74668288230896,
+      "learning_rate": 4.125922509225092e-06,
+      "loss": 1.5847,
+      "step": 635
+    },
+    {
+      "epoch": 0.791537025513379,
+      "grad_norm": 2.723123550415039,
+      "learning_rate": 4.1236162361623625e-06,
+      "loss": 1.5166,
+      "step": 636
+    },
+    {
+      "epoch": 0.7927815805849409,
+      "grad_norm": 2.7265713214874268,
+      "learning_rate": 4.121309963099631e-06,
+      "loss": 1.5954,
+      "step": 637
+    },
+    {
+      "epoch": 0.7940261356565028,
+      "grad_norm": 2.979126214981079,
+      "learning_rate": 4.1190036900369005e-06,
+      "loss": 1.5583,
+      "step": 638
+    },
+    {
+      "epoch": 0.7952706907280647,
+      "grad_norm": 2.844376802444458,
+      "learning_rate": 4.11669741697417e-06,
+      "loss": 1.5564,
+      "step": 639
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "grad_norm": 2.7583703994750977,
+      "learning_rate": 4.114391143911439e-06,
+      "loss": 1.5094,
+      "step": 640
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "eval_loss": 1.6986565589904785,
+      "eval_runtime": 47.3549,
+      "eval_samples_per_second": 21.117,
+      "eval_steps_per_second": 0.887,
+      "step": 640
+    },
+    {
+      "epoch": 0.7977598008711886,
+      "grad_norm": 2.8942224979400635,
+      "learning_rate": 4.112084870848709e-06,
+      "loss": 1.5504,
+      "step": 641
+    },
+    {
+      "epoch": 0.7990043559427504,
+      "grad_norm": 2.6590495109558105,
+      "learning_rate": 4.109778597785978e-06,
+      "loss": 1.5227,
+      "step": 642
+    },
+    {
+      "epoch": 0.8002489110143124,
+      "grad_norm": 2.5988378524780273,
+      "learning_rate": 4.107472324723248e-06,
+      "loss": 1.5054,
+      "step": 643
+    },
+    {
+      "epoch": 0.8014934660858744,
+      "grad_norm": 2.787335157394409,
+      "learning_rate": 4.105166051660517e-06,
+      "loss": 1.5844,
+      "step": 644
+    },
+    {
+      "epoch": 0.8027380211574362,
+      "grad_norm": 2.8202896118164062,
+      "learning_rate": 4.1028597785977864e-06,
+      "loss": 1.5596,
+      "step": 645
+    },
+    {
+      "epoch": 0.8039825762289982,
+      "grad_norm": 2.65376615524292,
+      "learning_rate": 4.100553505535056e-06,
+      "loss": 1.5893,
+      "step": 646
+    },
+    {
+      "epoch": 0.80522713130056,
+      "grad_norm": 2.5933308601379395,
+      "learning_rate": 4.098247232472325e-06,
+      "loss": 1.5403,
+      "step": 647
+    },
+    {
+      "epoch": 0.806471686372122,
+      "grad_norm": 2.777070999145508,
+      "learning_rate": 4.095940959409595e-06,
+      "loss": 1.5231,
+      "step": 648
+    },
+    {
+      "epoch": 0.8077162414436839,
+      "grad_norm": 2.6427664756774902,
+      "learning_rate": 4.093634686346864e-06,
+      "loss": 1.5937,
+      "step": 649
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "grad_norm": 2.651561737060547,
+      "learning_rate": 4.091328413284133e-06,
+      "loss": 1.4847,
+      "step": 650
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "eval_loss": 1.7089996337890625,
+      "eval_runtime": 47.4346,
+      "eval_samples_per_second": 21.082,
+      "eval_steps_per_second": 0.885,
+      "step": 650
+    },
+    {
+      "epoch": 0.8102053515868077,
+      "grad_norm": 2.844067335128784,
+      "learning_rate": 4.089022140221402e-06,
+      "loss": 1.488,
+      "step": 651
+    },
+    {
+      "epoch": 0.8114499066583696,
+      "grad_norm": 2.603480577468872,
+      "learning_rate": 4.086715867158672e-06,
+      "loss": 1.5248,
+      "step": 652
+    },
+    {
+      "epoch": 0.8126944617299315,
+      "grad_norm": 2.789322853088379,
+      "learning_rate": 4.084409594095941e-06,
+      "loss": 1.5119,
+      "step": 653
+    },
+    {
+      "epoch": 0.8139390168014935,
+      "grad_norm": 2.7183773517608643,
+      "learning_rate": 4.08210332103321e-06,
+      "loss": 1.5123,
+      "step": 654
+    },
+    {
+      "epoch": 0.8151835718730553,
+      "grad_norm": 2.80613374710083,
+      "learning_rate": 4.07979704797048e-06,
+      "loss": 1.5526,
+      "step": 655
+    },
+    {
+      "epoch": 0.8164281269446173,
+      "grad_norm": 2.641035795211792,
+      "learning_rate": 4.077490774907749e-06,
+      "loss": 1.6127,
+      "step": 656
+    },
+    {
+      "epoch": 0.8176726820161793,
+      "grad_norm": 2.773684024810791,
+      "learning_rate": 4.075184501845019e-06,
+      "loss": 1.5404,
+      "step": 657
+    },
+    {
+      "epoch": 0.8189172370877411,
+      "grad_norm": 2.6951382160186768,
+      "learning_rate": 4.072878228782288e-06,
+      "loss": 1.4928,
+      "step": 658
+    },
+    {
+      "epoch": 0.8201617921593031,
+      "grad_norm": 2.931771755218506,
+      "learning_rate": 4.0705719557195575e-06,
+      "loss": 1.5949,
+      "step": 659
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "grad_norm": 2.8786075115203857,
+      "learning_rate": 4.068265682656827e-06,
+      "loss": 1.5582,
+      "step": 660
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "eval_loss": 1.698564052581787,
+      "eval_runtime": 46.6299,
+      "eval_samples_per_second": 21.445,
+      "eval_steps_per_second": 0.901,
+      "step": 660
+    },
+    {
+      "epoch": 0.8226509023024269,
+      "grad_norm": 2.7402451038360596,
+      "learning_rate": 4.065959409594096e-06,
+      "loss": 1.5562,
+      "step": 661
+    },
+    {
+      "epoch": 0.8238954573739888,
+      "grad_norm": 2.7968692779541016,
+      "learning_rate": 4.063653136531366e-06,
+      "loss": 1.556,
+      "step": 662
+    },
+    {
+      "epoch": 0.8251400124455507,
+      "grad_norm": 2.8233132362365723,
+      "learning_rate": 4.061346863468635e-06,
+      "loss": 1.5527,
+      "step": 663
+    },
+    {
+      "epoch": 0.8263845675171126,
+      "grad_norm": 2.713390827178955,
+      "learning_rate": 4.059040590405905e-06,
+      "loss": 1.6012,
+      "step": 664
+    },
+    {
+      "epoch": 0.8276291225886746,
+      "grad_norm": 2.7783989906311035,
+      "learning_rate": 4.056734317343174e-06,
+      "loss": 1.5247,
+      "step": 665
+    },
+    {
+      "epoch": 0.8288736776602365,
+      "grad_norm": 2.808554172515869,
+      "learning_rate": 4.054428044280443e-06,
+      "loss": 1.5412,
+      "step": 666
+    },
+    {
+      "epoch": 0.8301182327317984,
+      "grad_norm": 2.8476831912994385,
+      "learning_rate": 4.052121771217712e-06,
+      "loss": 1.5307,
+      "step": 667
+    },
+    {
+      "epoch": 0.8313627878033603,
+      "grad_norm": 2.8287601470947266,
+      "learning_rate": 4.049815498154982e-06,
+      "loss": 1.6021,
+      "step": 668
+    },
+    {
+      "epoch": 0.8326073428749222,
+      "grad_norm": 2.975510835647583,
+      "learning_rate": 4.047509225092252e-06,
+      "loss": 1.5474,
+      "step": 669
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "grad_norm": 2.723236560821533,
+      "learning_rate": 4.04520295202952e-06,
+      "loss": 1.5242,
+      "step": 670
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "eval_loss": 1.7005239725112915,
+      "eval_runtime": 49.4504,
+      "eval_samples_per_second": 20.222,
+      "eval_steps_per_second": 0.849,
+      "step": 670
+    },
+    {
+      "epoch": 0.835096453018046,
+      "grad_norm": 2.7521748542785645,
+      "learning_rate": 4.04289667896679e-06,
+      "loss": 1.5467,
+      "step": 671
+    },
+    {
+      "epoch": 0.836341008089608,
+      "grad_norm": 2.7770044803619385,
+      "learning_rate": 4.04059040590406e-06,
+      "loss": 1.5407,
+      "step": 672
+    },
+    {
+      "epoch": 0.8375855631611698,
+      "grad_norm": 2.744323492050171,
+      "learning_rate": 4.038284132841329e-06,
+      "loss": 1.5422,
+      "step": 673
+    },
+    {
+      "epoch": 0.8388301182327318,
+      "grad_norm": 2.6699817180633545,
+      "learning_rate": 4.035977859778598e-06,
+      "loss": 1.5508,
+      "step": 674
+    },
+    {
+      "epoch": 0.8400746733042938,
+      "grad_norm": 2.705273151397705,
+      "learning_rate": 4.0336715867158674e-06,
+      "loss": 1.5099,
+      "step": 675
+    },
+    {
+      "epoch": 0.8413192283758556,
+      "grad_norm": 2.9485747814178467,
+      "learning_rate": 4.031365313653137e-06,
+      "loss": 1.5529,
+      "step": 676
+    },
+    {
+      "epoch": 0.8425637834474176,
+      "grad_norm": 3.1750423908233643,
+      "learning_rate": 4.029059040590406e-06,
+      "loss": 1.5894,
+      "step": 677
+    },
+    {
+      "epoch": 0.8438083385189795,
+      "grad_norm": 2.76448655128479,
+      "learning_rate": 4.026752767527676e-06,
+      "loss": 1.5752,
+      "step": 678
+    },
+    {
+      "epoch": 0.8450528935905414,
+      "grad_norm": 2.676708698272705,
+      "learning_rate": 4.024446494464945e-06,
+      "loss": 1.5487,
+      "step": 679
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "grad_norm": 2.6832051277160645,
+      "learning_rate": 4.0221402214022145e-06,
+      "loss": 1.5055,
+      "step": 680
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "eval_loss": 1.6945017576217651,
+      "eval_runtime": 43.9369,
+      "eval_samples_per_second": 22.76,
+      "eval_steps_per_second": 0.956,
+      "step": 680
+    },
+    {
+      "epoch": 0.8475420037336652,
+      "grad_norm": 2.6812336444854736,
+      "learning_rate": 4.019833948339484e-06,
+      "loss": 1.5315,
+      "step": 681
+    },
+    {
+      "epoch": 0.8487865588052271,
+      "grad_norm": 2.9394326210021973,
+      "learning_rate": 4.017527675276753e-06,
+      "loss": 1.5636,
+      "step": 682
+    },
+    {
+      "epoch": 0.8500311138767891,
+      "grad_norm": 2.944952964782715,
+      "learning_rate": 4.015221402214022e-06,
+      "loss": 1.56,
+      "step": 683
+    },
+    {
+      "epoch": 0.8512756689483509,
+      "grad_norm": 2.7475314140319824,
+      "learning_rate": 4.012915129151292e-06,
+      "loss": 1.5442,
+      "step": 684
+    },
+    {
+      "epoch": 0.8525202240199129,
+      "grad_norm": 3.0572879314422607,
+      "learning_rate": 4.010608856088562e-06,
+      "loss": 1.5023,
+      "step": 685
+    },
+    {
+      "epoch": 0.8537647790914747,
+      "grad_norm": 2.75365948677063,
+      "learning_rate": 4.00830258302583e-06,
+      "loss": 1.5414,
+      "step": 686
+    },
+    {
+      "epoch": 0.8550093341630367,
+      "grad_norm": 2.5714449882507324,
+      "learning_rate": 4.0059963099631e-06,
+      "loss": 1.5027,
+      "step": 687
+    },
+    {
+      "epoch": 0.8562538892345987,
+      "grad_norm": 2.755167007446289,
+      "learning_rate": 4.003690036900369e-06,
+      "loss": 1.5788,
+      "step": 688
+    },
+    {
+      "epoch": 0.8574984443061605,
+      "grad_norm": 2.798967123031616,
+      "learning_rate": 4.0013837638376385e-06,
+      "loss": 1.5162,
+      "step": 689
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "grad_norm": 2.803614854812622,
+      "learning_rate": 3.999077490774908e-06,
+      "loss": 1.5079,
+      "step": 690
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "eval_loss": 1.6924962997436523,
+      "eval_runtime": 44.47,
+      "eval_samples_per_second": 22.487,
+      "eval_steps_per_second": 0.944,
+      "step": 690
+    },
+    {
+      "epoch": 0.8599875544492844,
+      "grad_norm": 2.6898066997528076,
+      "learning_rate": 3.996771217712177e-06,
+      "loss": 1.5478,
+      "step": 691
+    },
+    {
+      "epoch": 0.8612321095208463,
+      "grad_norm": 2.6376144886016846,
+      "learning_rate": 3.994464944649447e-06,
+      "loss": 1.5362,
+      "step": 692
+    },
+    {
+      "epoch": 0.8624766645924082,
+      "grad_norm": 2.7240405082702637,
+      "learning_rate": 3.992158671586716e-06,
+      "loss": 1.5502,
+      "step": 693
+    },
+    {
+      "epoch": 0.8637212196639701,
+      "grad_norm": 2.929445266723633,
+      "learning_rate": 3.989852398523986e-06,
+      "loss": 1.5799,
+      "step": 694
+    },
+    {
+      "epoch": 0.864965774735532,
+      "grad_norm": 2.593223810195923,
+      "learning_rate": 3.987546125461255e-06,
+      "loss": 1.5352,
+      "step": 695
+    },
+    {
+      "epoch": 0.866210329807094,
+      "grad_norm": 2.7710154056549072,
+      "learning_rate": 3.9852398523985245e-06,
+      "loss": 1.5048,
+      "step": 696
+    },
+    {
+      "epoch": 0.8674548848786559,
+      "grad_norm": 2.7370848655700684,
+      "learning_rate": 3.982933579335794e-06,
+      "loss": 1.5233,
+      "step": 697
+    },
+    {
+      "epoch": 0.8686994399502178,
+      "grad_norm": 2.8632168769836426,
+      "learning_rate": 3.980627306273063e-06,
+      "loss": 1.5435,
+      "step": 698
+    },
+    {
+      "epoch": 0.8699439950217797,
+      "grad_norm": 2.752298593521118,
+      "learning_rate": 3.978321033210332e-06,
+      "loss": 1.5728,
+      "step": 699
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "grad_norm": 2.8345625400543213,
+      "learning_rate": 3.976014760147602e-06,
+      "loss": 1.5727,
+      "step": 700
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "eval_loss": 1.6855305433273315,
+      "eval_runtime": 43.3683,
+      "eval_samples_per_second": 23.058,
+      "eval_steps_per_second": 0.968,
+      "step": 700
+    },
+    {
+      "epoch": 0.8724331051649036,
+      "grad_norm": 2.8135335445404053,
+      "learning_rate": 3.973708487084872e-06,
+      "loss": 1.5086,
+      "step": 701
+    },
+    {
+      "epoch": 0.8736776602364654,
+      "grad_norm": 2.7416603565216064,
+      "learning_rate": 3.97140221402214e-06,
+      "loss": 1.545,
+      "step": 702
+    },
+    {
+      "epoch": 0.8749222153080274,
+      "grad_norm": 2.6902289390563965,
+      "learning_rate": 3.96909594095941e-06,
+      "loss": 1.5509,
+      "step": 703
+    },
+    {
+      "epoch": 0.8761667703795893,
+      "grad_norm": 2.6537084579467773,
+      "learning_rate": 3.966789667896679e-06,
+      "loss": 1.562,
+      "step": 704
+    },
+    {
+      "epoch": 0.8774113254511512,
+      "grad_norm": 2.6979284286499023,
+      "learning_rate": 3.964483394833948e-06,
+      "loss": 1.4829,
+      "step": 705
+    },
+    {
+      "epoch": 0.8786558805227132,
+      "grad_norm": 2.5795764923095703,
+      "learning_rate": 3.962177121771218e-06,
+      "loss": 1.4677,
+      "step": 706
+    },
+    {
+      "epoch": 0.879900435594275,
+      "grad_norm": 3.0185086727142334,
+      "learning_rate": 3.959870848708487e-06,
+      "loss": 1.5608,
+      "step": 707
+    },
+    {
+      "epoch": 0.881144990665837,
+      "grad_norm": 2.6458113193511963,
+      "learning_rate": 3.957564575645757e-06,
+      "loss": 1.5342,
+      "step": 708
+    },
+    {
+      "epoch": 0.8823895457373989,
+      "grad_norm": 2.7985665798187256,
+      "learning_rate": 3.955258302583026e-06,
+      "loss": 1.527,
+      "step": 709
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "grad_norm": 2.9069082736968994,
+      "learning_rate": 3.9529520295202955e-06,
+      "loss": 1.5156,
+      "step": 710
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "eval_loss": 1.6898822784423828,
+      "eval_runtime": 42.8749,
+      "eval_samples_per_second": 23.324,
+      "eval_steps_per_second": 0.98,
+      "step": 710
+    },
+    {
+      "epoch": 0.8848786558805227,
+      "grad_norm": 2.760143995285034,
+      "learning_rate": 3.950645756457565e-06,
+      "loss": 1.5124,
+      "step": 711
+    },
+    {
+      "epoch": 0.8861232109520847,
+      "grad_norm": 2.8085529804229736,
+      "learning_rate": 3.948339483394834e-06,
+      "loss": 1.5862,
+      "step": 712
+    },
+    {
+      "epoch": 0.8873677660236465,
+      "grad_norm": 2.909905195236206,
+      "learning_rate": 3.946033210332104e-06,
+      "loss": 1.5557,
+      "step": 713
+    },
+    {
+      "epoch": 0.8886123210952085,
+      "grad_norm": 2.826899290084839,
+      "learning_rate": 3.943726937269373e-06,
+      "loss": 1.5277,
+      "step": 714
+    },
+    {
+      "epoch": 0.8898568761667703,
+      "grad_norm": 2.6269052028656006,
+      "learning_rate": 3.941420664206642e-06,
+      "loss": 1.4819,
+      "step": 715
+    },
+    {
+      "epoch": 0.8911014312383323,
+      "grad_norm": 2.6687698364257812,
+      "learning_rate": 3.939114391143912e-06,
+      "loss": 1.5156,
+      "step": 716
+    },
+    {
+      "epoch": 0.8923459863098943,
+      "grad_norm": 2.9605629444122314,
+      "learning_rate": 3.9368081180811815e-06,
+      "loss": 1.5358,
+      "step": 717
+    },
+    {
+      "epoch": 0.8935905413814561,
+      "grad_norm": 3.0367166996002197,
+      "learning_rate": 3.934501845018451e-06,
+      "loss": 1.5847,
+      "step": 718
+    },
+    {
+      "epoch": 0.8948350964530181,
+      "grad_norm": 2.528796434402466,
+      "learning_rate": 3.9321955719557195e-06,
+      "loss": 1.4851,
+      "step": 719
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "grad_norm": 2.7116613388061523,
+      "learning_rate": 3.929889298892989e-06,
+      "loss": 1.4706,
+      "step": 720
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "eval_loss": 1.6845883131027222,
+      "eval_runtime": 44.4734,
+      "eval_samples_per_second": 22.485,
+      "eval_steps_per_second": 0.944,
+      "step": 720
+    },
+    {
+      "epoch": 0.8973242065961419,
+      "grad_norm": 2.771864414215088,
+      "learning_rate": 3.927583025830259e-06,
+      "loss": 1.5444,
+      "step": 721
+    },
+    {
+      "epoch": 0.8985687616677038,
+      "grad_norm": 2.773746967315674,
+      "learning_rate": 3.925276752767528e-06,
+      "loss": 1.564,
+      "step": 722
+    },
+    {
+      "epoch": 0.8998133167392657,
+      "grad_norm": 2.7069854736328125,
+      "learning_rate": 3.922970479704797e-06,
+      "loss": 1.5601,
+      "step": 723
+    },
+    {
+      "epoch": 0.9010578718108276,
+      "grad_norm": 2.840261697769165,
+      "learning_rate": 3.920664206642067e-06,
+      "loss": 1.5733,
+      "step": 724
+    },
+    {
+      "epoch": 0.9023024268823896,
+      "grad_norm": 2.709897041320801,
+      "learning_rate": 3.918357933579336e-06,
+      "loss": 1.528,
+      "step": 725
+    },
+    {
+      "epoch": 0.9035469819539514,
+      "grad_norm": 2.662367343902588,
+      "learning_rate": 3.9160516605166055e-06,
+      "loss": 1.5532,
+      "step": 726
+    },
+    {
+      "epoch": 0.9047915370255134,
+      "grad_norm": 2.6775717735290527,
+      "learning_rate": 3.913745387453875e-06,
+      "loss": 1.5464,
+      "step": 727
+    },
+    {
+      "epoch": 0.9060360920970753,
+      "grad_norm": 2.617842674255371,
+      "learning_rate": 3.911439114391144e-06,
+      "loss": 1.4539,
+      "step": 728
+    },
+    {
+      "epoch": 0.9072806471686372,
+      "grad_norm": 2.668649435043335,
+      "learning_rate": 3.909132841328414e-06,
+      "loss": 1.5627,
+      "step": 729
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "grad_norm": 2.5331950187683105,
+      "learning_rate": 3.906826568265683e-06,
+      "loss": 1.5165,
+      "step": 730
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "eval_loss": 1.6860331296920776,
+      "eval_runtime": 46.094,
+      "eval_samples_per_second": 21.695,
+      "eval_steps_per_second": 0.911,
+      "step": 730
+    },
+    {
+      "epoch": 0.909769757311761,
+      "grad_norm": 2.602492332458496,
+      "learning_rate": 3.9045202952029526e-06,
+      "loss": 1.5145,
+      "step": 731
+    },
+    {
+      "epoch": 0.911014312383323,
+      "grad_norm": 2.632782459259033,
+      "learning_rate": 3.902214022140222e-06,
+      "loss": 1.4413,
+      "step": 732
+    },
+    {
+      "epoch": 0.9122588674548848,
+      "grad_norm": 2.6025912761688232,
+      "learning_rate": 3.899907749077491e-06,
+      "loss": 1.5263,
+      "step": 733
+    },
+    {
+      "epoch": 0.9135034225264468,
+      "grad_norm": 2.770116090774536,
+      "learning_rate": 3.897601476014761e-06,
+      "loss": 1.5514,
+      "step": 734
+    },
+    {
+      "epoch": 0.9147479775980087,
+      "grad_norm": 2.7822961807250977,
+      "learning_rate": 3.895295202952029e-06,
+      "loss": 1.5296,
+      "step": 735
+    },
+    {
+      "epoch": 0.9159925326695706,
+      "grad_norm": 2.6689720153808594,
+      "learning_rate": 3.892988929889299e-06,
+      "loss": 1.4552,
+      "step": 736
+    },
+    {
+      "epoch": 0.9172370877411326,
+      "grad_norm": 2.8021798133850098,
+      "learning_rate": 3.890682656826569e-06,
+      "loss": 1.5308,
+      "step": 737
+    },
+    {
+      "epoch": 0.9184816428126945,
+      "grad_norm": 2.575313091278076,
+      "learning_rate": 3.888376383763838e-06,
+      "loss": 1.4807,
+      "step": 738
+    },
+    {
+      "epoch": 0.9197261978842564,
+      "grad_norm": 2.5955779552459717,
+      "learning_rate": 3.886070110701107e-06,
+      "loss": 1.4222,
+      "step": 739
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "grad_norm": 2.647939920425415,
+      "learning_rate": 3.8837638376383765e-06,
+      "loss": 1.5285,
+      "step": 740
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "eval_loss": 1.6907480955123901,
+      "eval_runtime": 45.7678,
+      "eval_samples_per_second": 21.849,
+      "eval_steps_per_second": 0.918,
+      "step": 740
+    },
+    {
+      "epoch": 0.9222153080273802,
+      "grad_norm": 2.7664523124694824,
+      "learning_rate": 3.881457564575646e-06,
+      "loss": 1.5349,
+      "step": 741
+    },
+    {
+      "epoch": 0.9234598630989421,
+      "grad_norm": 2.5770998001098633,
+      "learning_rate": 3.879151291512915e-06,
+      "loss": 1.5071,
+      "step": 742
+    },
+    {
+      "epoch": 0.9247044181705041,
+      "grad_norm": 2.502567768096924,
+      "learning_rate": 3.876845018450185e-06,
+      "loss": 1.492,
+      "step": 743
+    },
+    {
+      "epoch": 0.9259489732420659,
+      "grad_norm": 2.6531119346618652,
+      "learning_rate": 3.874538745387454e-06,
+      "loss": 1.5045,
+      "step": 744
+    },
+    {
+      "epoch": 0.9271935283136279,
+      "grad_norm": 2.7343876361846924,
+      "learning_rate": 3.872232472324724e-06,
+      "loss": 1.5194,
+      "step": 745
+    },
+    {
+      "epoch": 0.9284380833851898,
+      "grad_norm": 2.842313528060913,
+      "learning_rate": 3.869926199261993e-06,
+      "loss": 1.5588,
+      "step": 746
+    },
+    {
+      "epoch": 0.9296826384567517,
+      "grad_norm": 2.718245267868042,
+      "learning_rate": 3.8676199261992625e-06,
+      "loss": 1.5252,
+      "step": 747
+    },
+    {
+      "epoch": 0.9309271935283137,
+      "grad_norm": 2.695392370223999,
+      "learning_rate": 3.865313653136532e-06,
+      "loss": 1.5596,
+      "step": 748
+    },
+    {
+      "epoch": 0.9321717485998755,
+      "grad_norm": 2.7859959602355957,
+      "learning_rate": 3.863007380073801e-06,
+      "loss": 1.5151,
+      "step": 749
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "grad_norm": 2.5199174880981445,
+      "learning_rate": 3.860701107011071e-06,
+      "loss": 1.4948,
+      "step": 750
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "eval_loss": 1.6868674755096436,
+      "eval_runtime": 46.4115,
+      "eval_samples_per_second": 21.546,
+      "eval_steps_per_second": 0.905,
+      "step": 750
+    },
+    {
+      "epoch": 0.9346608587429994,
+      "grad_norm": 2.8194046020507812,
+      "learning_rate": 3.858394833948339e-06,
+      "loss": 1.5144,
+      "step": 751
+    },
+    {
+      "epoch": 0.9359054138145613,
+      "grad_norm": 2.810380697250366,
+      "learning_rate": 3.856088560885609e-06,
+      "loss": 1.4998,
+      "step": 752
+    },
+    {
+      "epoch": 0.9371499688861232,
+      "grad_norm": 2.6906750202178955,
+      "learning_rate": 3.853782287822879e-06,
+      "loss": 1.4748,
+      "step": 753
+    },
+    {
+      "epoch": 0.9383945239576851,
+      "grad_norm": 2.9030277729034424,
+      "learning_rate": 3.851476014760148e-06,
+      "loss": 1.615,
+      "step": 754
+    },
+    {
+      "epoch": 0.939639079029247,
+      "grad_norm": 2.8690176010131836,
+      "learning_rate": 3.849169741697417e-06,
+      "loss": 1.5083,
+      "step": 755
+    },
+    {
+      "epoch": 0.940883634100809,
+      "grad_norm": 2.669646978378296,
+      "learning_rate": 3.8468634686346865e-06,
+      "loss": 1.5572,
+      "step": 756
+    },
+    {
+      "epoch": 0.9421281891723708,
+      "grad_norm": 2.6517696380615234,
+      "learning_rate": 3.844557195571956e-06,
+      "loss": 1.5142,
+      "step": 757
+    },
+    {
+      "epoch": 0.9433727442439328,
+      "grad_norm": 2.5524444580078125,
+      "learning_rate": 3.842250922509225e-06,
+      "loss": 1.4924,
+      "step": 758
+    },
+    {
+      "epoch": 0.9446172993154948,
+      "grad_norm": 2.6532633304595947,
+      "learning_rate": 3.839944649446495e-06,
+      "loss": 1.484,
+      "step": 759
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "grad_norm": 2.7779057025909424,
+      "learning_rate": 3.837638376383764e-06,
+      "loss": 1.5106,
+      "step": 760
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "eval_loss": 1.6845752000808716,
+      "eval_runtime": 50.3415,
+      "eval_samples_per_second": 19.864,
+      "eval_steps_per_second": 0.834,
+      "step": 760
+    },
+    {
+      "epoch": 0.9471064094586186,
+      "grad_norm": 2.6192541122436523,
+      "learning_rate": 3.8353321033210336e-06,
+      "loss": 1.5454,
+      "step": 761
+    },
+    {
+      "epoch": 0.9483509645301804,
+      "grad_norm": 2.561861991882324,
+      "learning_rate": 3.833025830258303e-06,
+      "loss": 1.5141,
+      "step": 762
+    },
+    {
+      "epoch": 0.9495955196017424,
+      "grad_norm": 2.661829948425293,
+      "learning_rate": 3.830719557195572e-06,
+      "loss": 1.4997,
+      "step": 763
+    },
+    {
+      "epoch": 0.9508400746733043,
+      "grad_norm": 2.725275754928589,
+      "learning_rate": 3.828413284132842e-06,
+      "loss": 1.5224,
+      "step": 764
+    },
+    {
+      "epoch": 0.9520846297448662,
+      "grad_norm": 2.5664193630218506,
+      "learning_rate": 3.826107011070111e-06,
+      "loss": 1.5073,
+      "step": 765
+    },
+    {
+      "epoch": 0.9533291848164281,
+      "grad_norm": 2.595189332962036,
+      "learning_rate": 3.823800738007381e-06,
+      "loss": 1.5379,
+      "step": 766
+    },
+    {
+      "epoch": 0.95457373988799,
+      "grad_norm": 2.738060235977173,
+      "learning_rate": 3.821494464944649e-06,
+      "loss": 1.4814,
+      "step": 767
+    },
+    {
+      "epoch": 0.955818294959552,
+      "grad_norm": 2.601071357727051,
+      "learning_rate": 3.819188191881919e-06,
+      "loss": 1.4812,
+      "step": 768
+    },
+    {
+      "epoch": 0.9570628500311139,
+      "grad_norm": 2.657212257385254,
+      "learning_rate": 3.816881918819189e-06,
+      "loss": 1.4748,
+      "step": 769
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "grad_norm": 2.7673351764678955,
+      "learning_rate": 3.814575645756458e-06,
+      "loss": 1.5535,
+      "step": 770
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "eval_loss": 1.677243947982788,
+      "eval_runtime": 51.5537,
+      "eval_samples_per_second": 19.397,
+      "eval_steps_per_second": 0.815,
+      "step": 770
+    },
+    {
+      "epoch": 0.9595519601742377,
+      "grad_norm": 2.760890007019043,
+      "learning_rate": 3.812269372693727e-06,
+      "loss": 1.5654,
+      "step": 771
+    },
+    {
+      "epoch": 0.9607965152457997,
+      "grad_norm": 2.6934309005737305,
+      "learning_rate": 3.809963099630997e-06,
+      "loss": 1.5149,
+      "step": 772
+    },
+    {
+      "epoch": 0.9620410703173615,
+      "grad_norm": 2.729950428009033,
+      "learning_rate": 3.8076568265682662e-06,
+      "loss": 1.5579,
+      "step": 773
+    },
+    {
+      "epoch": 0.9632856253889235,
+      "grad_norm": 2.732926607131958,
+      "learning_rate": 3.8053505535055352e-06,
+      "loss": 1.5179,
+      "step": 774
+    },
+    {
+      "epoch": 0.9645301804604853,
+      "grad_norm": 2.745391368865967,
+      "learning_rate": 3.8030442804428046e-06,
+      "loss": 1.5124,
+      "step": 775
+    },
+    {
+      "epoch": 0.9657747355320473,
+      "grad_norm": 2.8050146102905273,
+      "learning_rate": 3.800738007380074e-06,
+      "loss": 1.519,
+      "step": 776
+    },
+    {
+      "epoch": 0.9670192906036092,
+      "grad_norm": 2.697171449661255,
+      "learning_rate": 3.798431734317343e-06,
+      "loss": 1.5219,
+      "step": 777
+    },
+    {
+      "epoch": 0.9682638456751711,
+      "grad_norm": 2.6375980377197266,
+      "learning_rate": 3.796125461254613e-06,
+      "loss": 1.5345,
+      "step": 778
+    },
+    {
+      "epoch": 0.9695084007467331,
+      "grad_norm": 2.586636781692505,
+      "learning_rate": 3.7938191881918823e-06,
+      "loss": 1.5182,
+      "step": 779
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "grad_norm": 2.626453399658203,
+      "learning_rate": 3.7915129151291518e-06,
+      "loss": 1.4817,
+      "step": 780
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "eval_loss": 1.6734713315963745,
+      "eval_runtime": 48.1436,
+      "eval_samples_per_second": 20.771,
+      "eval_steps_per_second": 0.872,
+      "step": 780
+    },
+    {
+      "epoch": 0.9719975108898569,
+      "grad_norm": 2.579970121383667,
+      "learning_rate": 3.7892066420664208e-06,
+      "loss": 1.4981,
+      "step": 781
+    },
+    {
+      "epoch": 0.9732420659614188,
+      "grad_norm": 2.6834843158721924,
+      "learning_rate": 3.7869003690036906e-06,
+      "loss": 1.5044,
+      "step": 782
+    },
+    {
+      "epoch": 0.9744866210329807,
+      "grad_norm": 2.586982488632202,
+      "learning_rate": 3.78459409594096e-06,
+      "loss": 1.5079,
+      "step": 783
+    },
+    {
+      "epoch": 0.9757311761045426,
+      "grad_norm": 2.910027503967285,
+      "learning_rate": 3.782287822878229e-06,
+      "loss": 1.5695,
+      "step": 784
+    },
+    {
+      "epoch": 0.9769757311761046,
+      "grad_norm": 2.7088494300842285,
+      "learning_rate": 3.7799815498154984e-06,
+      "loss": 1.4986,
+      "step": 785
+    },
+    {
+      "epoch": 0.9782202862476664,
+      "grad_norm": 2.581325054168701,
+      "learning_rate": 3.777675276752768e-06,
+      "loss": 1.5137,
+      "step": 786
+    },
+    {
+      "epoch": 0.9794648413192284,
+      "grad_norm": 2.700709819793701,
+      "learning_rate": 3.775369003690037e-06,
+      "loss": 1.4718,
+      "step": 787
+    },
+    {
+      "epoch": 0.9807093963907902,
+      "grad_norm": 2.6712708473205566,
+      "learning_rate": 3.7730627306273067e-06,
+      "loss": 1.5208,
+      "step": 788
+    },
+    {
+      "epoch": 0.9819539514623522,
+      "grad_norm": 2.6799817085266113,
+      "learning_rate": 3.770756457564576e-06,
+      "loss": 1.5374,
+      "step": 789
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "grad_norm": 2.618988037109375,
+      "learning_rate": 3.768450184501845e-06,
+      "loss": 1.4997,
+      "step": 790
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "eval_loss": 1.6860820055007935,
+      "eval_runtime": 51.7654,
+      "eval_samples_per_second": 19.318,
+      "eval_steps_per_second": 0.811,
+      "step": 790
+    },
+    {
+      "epoch": 0.984443061605476,
+      "grad_norm": 2.6899526119232178,
+      "learning_rate": 3.7661439114391146e-06,
+      "loss": 1.5216,
+      "step": 791
+    },
+    {
+      "epoch": 0.985687616677038,
+      "grad_norm": 2.8003487586975098,
+      "learning_rate": 3.763837638376384e-06,
+      "loss": 1.557,
+      "step": 792
+    },
+    {
+      "epoch": 0.9869321717485999,
+      "grad_norm": 2.744536876678467,
+      "learning_rate": 3.761531365313654e-06,
+      "loss": 1.5355,
+      "step": 793
+    },
+    {
+      "epoch": 0.9881767268201618,
+      "grad_norm": 2.587250232696533,
+      "learning_rate": 3.759225092250923e-06,
+      "loss": 1.5259,
+      "step": 794
+    },
+    {
+      "epoch": 0.9894212818917237,
+      "grad_norm": 2.616291046142578,
+      "learning_rate": 3.7569188191881922e-06,
+      "loss": 1.5176,
+      "step": 795
+    },
+    {
+      "epoch": 0.9906658369632856,
+      "grad_norm": 2.6410577297210693,
+      "learning_rate": 3.7546125461254617e-06,
+      "loss": 1.511,
+      "step": 796
+    },
+    {
+      "epoch": 0.9919103920348475,
+      "grad_norm": 2.577373504638672,
+      "learning_rate": 3.7523062730627307e-06,
+      "loss": 1.4715,
+      "step": 797
+    },
+    {
+      "epoch": 0.9931549471064095,
+      "grad_norm": 2.67305326461792,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 1.4828,
+      "step": 798
+    },
+    {
+      "epoch": 0.9943995021779714,
+      "grad_norm": 3.1631500720977783,
+      "learning_rate": 3.74769372693727e-06,
+      "loss": 1.5602,
+      "step": 799
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "grad_norm": 2.9222350120544434,
+      "learning_rate": 3.745387453874539e-06,
+      "loss": 1.4814,
+      "step": 800
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "eval_loss": 1.6810544729232788,
+      "eval_runtime": 51.1262,
+      "eval_samples_per_second": 19.559,
+      "eval_steps_per_second": 0.821,
+      "step": 800
+    },
+    {
+      "epoch": 0.9968886123210952,
+      "grad_norm": 2.884312629699707,
+      "learning_rate": 3.7430811808118084e-06,
+      "loss": 1.4553,
+      "step": 801
+    },
+    {
+      "epoch": 0.9981331673926571,
+      "grad_norm": 2.712655782699585,
+      "learning_rate": 3.7407749077490778e-06,
+      "loss": 1.5275,
+      "step": 802
+    },
+    {
+      "epoch": 0.9993777224642191,
+      "grad_norm": 2.8700571060180664,
+      "learning_rate": 3.7384686346863468e-06,
+      "loss": 1.4923,
+      "step": 803
+    },
+    {
+      "epoch": 1.000622277535781,
+      "grad_norm": 2.7779245376586914,
+      "learning_rate": 3.7361623616236166e-06,
+      "loss": 1.5224,
+      "step": 804
+    },
+    {
+      "epoch": 1.0018668326073428,
+      "grad_norm": 2.7679810523986816,
+      "learning_rate": 3.733856088560886e-06,
+      "loss": 1.4977,
+      "step": 805
+    },
+    {
+      "epoch": 1.0031113876789048,
+      "grad_norm": 2.797023296356201,
+      "learning_rate": 3.7315498154981555e-06,
+      "loss": 1.4396,
+      "step": 806
+    },
+    {
+      "epoch": 1.0043559427504667,
+      "grad_norm": 3.007962465286255,
+      "learning_rate": 3.7292435424354245e-06,
+      "loss": 1.5015,
+      "step": 807
+    },
+    {
+      "epoch": 1.0056004978220285,
+      "grad_norm": 3.127639055252075,
+      "learning_rate": 3.726937269372694e-06,
+      "loss": 1.5176,
+      "step": 808
+    },
+    {
+      "epoch": 1.0068450528935906,
+      "grad_norm": 2.9001357555389404,
+      "learning_rate": 3.7246309963099637e-06,
+      "loss": 1.4735,
+      "step": 809
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "grad_norm": 2.8245413303375244,
+      "learning_rate": 3.7223247232472327e-06,
+      "loss": 1.486,
+      "step": 810
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "eval_loss": 1.6835130453109741,
+      "eval_runtime": 44.5682,
+      "eval_samples_per_second": 22.438,
+      "eval_steps_per_second": 0.942,
+      "step": 810
+    },
+    {
+      "epoch": 1.0093341630367143,
+      "grad_norm": 2.780172824859619,
+      "learning_rate": 3.720018450184502e-06,
+      "loss": 1.4437,
+      "step": 811
+    },
+    {
+      "epoch": 1.0105787181082764,
+      "grad_norm": 2.6013996601104736,
+      "learning_rate": 3.7177121771217716e-06,
+      "loss": 1.4529,
+      "step": 812
+    },
+    {
+      "epoch": 1.0118232731798382,
+      "grad_norm": 2.831015110015869,
+      "learning_rate": 3.7154059040590406e-06,
+      "loss": 1.4825,
+      "step": 813
+    },
+    {
+      "epoch": 1.0130678282514,
+      "grad_norm": 2.8341829776763916,
+      "learning_rate": 3.71309963099631e-06,
+      "loss": 1.4736,
+      "step": 814
+    },
+    {
+      "epoch": 1.0143123833229621,
+      "grad_norm": 2.769540309906006,
+      "learning_rate": 3.71079335793358e-06,
+      "loss": 1.4724,
+      "step": 815
+    },
+    {
+      "epoch": 1.015556938394524,
+      "grad_norm": 3.0083069801330566,
+      "learning_rate": 3.708487084870849e-06,
+      "loss": 1.4985,
+      "step": 816
+    },
+    {
+      "epoch": 1.0168014934660858,
+      "grad_norm": 3.052168607711792,
+      "learning_rate": 3.7061808118081183e-06,
+      "loss": 1.5193,
+      "step": 817
+    },
+    {
+      "epoch": 1.0180460485376477,
+      "grad_norm": 2.924877405166626,
+      "learning_rate": 3.7038745387453877e-06,
+      "loss": 1.5219,
+      "step": 818
+    },
+    {
+      "epoch": 1.0192906036092098,
+      "grad_norm": 3.150123357772827,
+      "learning_rate": 3.7015682656826576e-06,
+      "loss": 1.5026,
+      "step": 819
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "grad_norm": 2.881655216217041,
+      "learning_rate": 3.6992619926199266e-06,
+      "loss": 1.5011,
+      "step": 820
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "eval_loss": 1.6745303869247437,
+      "eval_runtime": 50.6791,
+      "eval_samples_per_second": 19.732,
+      "eval_steps_per_second": 0.829,
+      "step": 820
+    },
+    {
+      "epoch": 1.0217797137523335,
+      "grad_norm": 2.824249505996704,
+      "learning_rate": 3.696955719557196e-06,
+      "loss": 1.4642,
+      "step": 821
+    },
+    {
+      "epoch": 1.0230242688238955,
+      "grad_norm": 2.859144926071167,
+      "learning_rate": 3.6946494464944654e-06,
+      "loss": 1.4681,
+      "step": 822
+    },
+    {
+      "epoch": 1.0242688238954574,
+      "grad_norm": 2.90312123298645,
+      "learning_rate": 3.6923431734317344e-06,
+      "loss": 1.4926,
+      "step": 823
+    },
+    {
+      "epoch": 1.0255133789670192,
+      "grad_norm": 2.8340659141540527,
+      "learning_rate": 3.690036900369004e-06,
+      "loss": 1.4834,
+      "step": 824
+    },
+    {
+      "epoch": 1.0267579340385813,
+      "grad_norm": 2.9743151664733887,
+      "learning_rate": 3.6877306273062737e-06,
+      "loss": 1.4574,
+      "step": 825
+    },
+    {
+      "epoch": 1.0280024891101431,
+      "grad_norm": 3.032179594039917,
+      "learning_rate": 3.6854243542435427e-06,
+      "loss": 1.5571,
+      "step": 826
+    },
+    {
+      "epoch": 1.029247044181705,
+      "grad_norm": 2.816826343536377,
+      "learning_rate": 3.683118081180812e-06,
+      "loss": 1.455,
+      "step": 827
+    },
+    {
+      "epoch": 1.030491599253267,
+      "grad_norm": 3.046696186065674,
+      "learning_rate": 3.6808118081180815e-06,
+      "loss": 1.4871,
+      "step": 828
+    },
+    {
+      "epoch": 1.031736154324829,
+      "grad_norm": 2.8517391681671143,
+      "learning_rate": 3.678505535055351e-06,
+      "loss": 1.5292,
+      "step": 829
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "grad_norm": 2.752811908721924,
+      "learning_rate": 3.67619926199262e-06,
+      "loss": 1.5091,
+      "step": 830
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "eval_loss": 1.6692287921905518,
+      "eval_runtime": 50.8347,
+      "eval_samples_per_second": 19.672,
+      "eval_steps_per_second": 0.826,
+      "step": 830
+    },
+    {
+      "epoch": 1.0342252644679526,
+      "grad_norm": 2.9220356941223145,
+      "learning_rate": 3.6738929889298898e-06,
+      "loss": 1.4247,
+      "step": 831
+    },
+    {
+      "epoch": 1.0354698195395147,
+      "grad_norm": 2.8903002738952637,
+      "learning_rate": 3.671586715867159e-06,
+      "loss": 1.5595,
+      "step": 832
+    },
+    {
+      "epoch": 1.0367143746110765,
+      "grad_norm": 2.6495542526245117,
+      "learning_rate": 3.669280442804428e-06,
+      "loss": 1.4076,
+      "step": 833
+    },
+    {
+      "epoch": 1.0379589296826384,
+      "grad_norm": 2.880809783935547,
+      "learning_rate": 3.6669741697416976e-06,
+      "loss": 1.5285,
+      "step": 834
+    },
+    {
+      "epoch": 1.0392034847542004,
+      "grad_norm": 2.919142007827759,
+      "learning_rate": 3.6646678966789675e-06,
+      "loss": 1.5165,
+      "step": 835
+    },
+    {
+      "epoch": 1.0404480398257623,
+      "grad_norm": 3.1180522441864014,
+      "learning_rate": 3.6623616236162365e-06,
+      "loss": 1.5224,
+      "step": 836
+    },
+    {
+      "epoch": 1.0416925948973241,
+      "grad_norm": 2.767709255218506,
+      "learning_rate": 3.660055350553506e-06,
+      "loss": 1.4859,
+      "step": 837
+    },
+    {
+      "epoch": 1.0429371499688862,
+      "grad_norm": 2.708828926086426,
+      "learning_rate": 3.6577490774907753e-06,
+      "loss": 1.4803,
+      "step": 838
+    },
+    {
+      "epoch": 1.044181705040448,
+      "grad_norm": 2.688359260559082,
+      "learning_rate": 3.6554428044280443e-06,
+      "loss": 1.4571,
+      "step": 839
+    },
+    {
+      "epoch": 1.04542626011201,
+      "grad_norm": 2.691812753677368,
+      "learning_rate": 3.6531365313653137e-06,
+      "loss": 1.5043,
+      "step": 840
+    },
+    {
+      "epoch": 1.04542626011201,
+      "eval_loss": 1.6769559383392334,
+      "eval_runtime": 49.6233,
+      "eval_samples_per_second": 20.152,
+      "eval_steps_per_second": 0.846,
+      "step": 840
+    },
+    {
+      "epoch": 1.046670815183572,
+      "grad_norm": 2.686372995376587,
+      "learning_rate": 3.6508302583025836e-06,
+      "loss": 1.5221,
+      "step": 841
+    },
+    {
+      "epoch": 1.0479153702551338,
+      "grad_norm": 2.742859125137329,
+      "learning_rate": 3.648523985239853e-06,
+      "loss": 1.4685,
+      "step": 842
+    },
+    {
+      "epoch": 1.0491599253266957,
+      "grad_norm": 2.874018907546997,
+      "learning_rate": 3.646217712177122e-06,
+      "loss": 1.4813,
+      "step": 843
+    },
+    {
+      "epoch": 1.0504044803982575,
+      "grad_norm": 2.8159990310668945,
+      "learning_rate": 3.6439114391143914e-06,
+      "loss": 1.4421,
+      "step": 844
+    },
+    {
+      "epoch": 1.0516490354698196,
+      "grad_norm": 2.714787006378174,
+      "learning_rate": 3.641605166051661e-06,
+      "loss": 1.4879,
+      "step": 845
+    },
+    {
+      "epoch": 1.0528935905413814,
+      "grad_norm": 2.885148525238037,
+      "learning_rate": 3.63929889298893e-06,
+      "loss": 1.4725,
+      "step": 846
+    },
+    {
+      "epoch": 1.0541381456129433,
+      "grad_norm": 2.7272610664367676,
+      "learning_rate": 3.6369926199261997e-06,
+      "loss": 1.4276,
+      "step": 847
+    },
+    {
+      "epoch": 1.0553827006845053,
+      "grad_norm": 2.9114978313446045,
+      "learning_rate": 3.634686346863469e-06,
+      "loss": 1.4194,
+      "step": 848
+    },
+    {
+      "epoch": 1.0566272557560672,
+      "grad_norm": 3.019313335418701,
+      "learning_rate": 3.632380073800738e-06,
+      "loss": 1.5266,
+      "step": 849
+    },
+    {
+      "epoch": 1.057871810827629,
+      "grad_norm": 2.714751958847046,
+      "learning_rate": 3.6300738007380075e-06,
+      "loss": 1.4593,
+      "step": 850
+    },
+    {
+      "epoch": 1.057871810827629,
+      "eval_loss": 1.6760298013687134,
+      "eval_runtime": 52.1681,
+      "eval_samples_per_second": 19.169,
+      "eval_steps_per_second": 0.805,
+      "step": 850
+    },
+    {
+      "epoch": 1.0591163658991911,
+      "grad_norm": 3.0507936477661133,
+      "learning_rate": 3.627767527675277e-06,
+      "loss": 1.5064,
+      "step": 851
+    },
+    {
+      "epoch": 1.060360920970753,
+      "grad_norm": 2.8116295337677,
+      "learning_rate": 3.625461254612546e-06,
+      "loss": 1.5218,
+      "step": 852
+    },
+    {
+      "epoch": 1.0616054760423148,
+      "grad_norm": 2.8801465034484863,
+      "learning_rate": 3.623154981549816e-06,
+      "loss": 1.4641,
+      "step": 853
+    },
+    {
+      "epoch": 1.0628500311138769,
+      "grad_norm": 3.0581862926483154,
+      "learning_rate": 3.6208487084870852e-06,
+      "loss": 1.483,
+      "step": 854
+    },
+    {
+      "epoch": 1.0640945861854387,
+      "grad_norm": 2.898780584335327,
+      "learning_rate": 3.6185424354243547e-06,
+      "loss": 1.4623,
+      "step": 855
+    },
+    {
+      "epoch": 1.0653391412570006,
+      "grad_norm": 2.971820116043091,
+      "learning_rate": 3.6162361623616237e-06,
+      "loss": 1.4455,
+      "step": 856
+    },
+    {
+      "epoch": 1.0665836963285624,
+      "grad_norm": 3.0021913051605225,
+      "learning_rate": 3.6139298892988935e-06,
+      "loss": 1.4878,
+      "step": 857
+    },
+    {
+      "epoch": 1.0678282514001245,
+      "grad_norm": 3.0697457790374756,
+      "learning_rate": 3.611623616236163e-06,
+      "loss": 1.5008,
+      "step": 858
+    },
+    {
+      "epoch": 1.0690728064716863,
+      "grad_norm": 3.0787923336029053,
+      "learning_rate": 3.609317343173432e-06,
+      "loss": 1.4612,
+      "step": 859
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "grad_norm": 2.789339780807495,
+      "learning_rate": 3.6070110701107014e-06,
+      "loss": 1.4287,
+      "step": 860
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "eval_loss": 1.6747506856918335,
+      "eval_runtime": 51.4912,
+      "eval_samples_per_second": 19.421,
+      "eval_steps_per_second": 0.816,
+      "step": 860
+    },
+    {
+      "epoch": 1.0715619166148103,
+      "grad_norm": 2.9622788429260254,
+      "learning_rate": 3.6047047970479708e-06,
+      "loss": 1.4905,
+      "step": 861
+    },
+    {
+      "epoch": 1.072806471686372,
+      "grad_norm": 3.0675268173217773,
+      "learning_rate": 3.6023985239852398e-06,
+      "loss": 1.461,
+      "step": 862
+    },
+    {
+      "epoch": 1.074051026757934,
+      "grad_norm": 2.820996046066284,
+      "learning_rate": 3.6000922509225096e-06,
+      "loss": 1.4948,
+      "step": 863
+    },
+    {
+      "epoch": 1.075295581829496,
+      "grad_norm": 2.833761215209961,
+      "learning_rate": 3.597785977859779e-06,
+      "loss": 1.4683,
+      "step": 864
+    },
+    {
+      "epoch": 1.0765401369010579,
+      "grad_norm": 2.917313814163208,
+      "learning_rate": 3.595479704797048e-06,
+      "loss": 1.5191,
+      "step": 865
+    },
+    {
+      "epoch": 1.0777846919726197,
+      "grad_norm": 2.876166820526123,
+      "learning_rate": 3.5931734317343175e-06,
+      "loss": 1.4854,
+      "step": 866
+    },
+    {
+      "epoch": 1.0790292470441818,
+      "grad_norm": 3.139049768447876,
+      "learning_rate": 3.590867158671587e-06,
+      "loss": 1.5134,
+      "step": 867
+    },
+    {
+      "epoch": 1.0802738021157436,
+      "grad_norm": 2.9956753253936768,
+      "learning_rate": 3.5885608856088567e-06,
+      "loss": 1.5003,
+      "step": 868
+    },
+    {
+      "epoch": 1.0815183571873055,
+      "grad_norm": 2.957181930541992,
+      "learning_rate": 3.5862546125461257e-06,
+      "loss": 1.4588,
+      "step": 869
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "grad_norm": 2.8576438426971436,
+      "learning_rate": 3.583948339483395e-06,
+      "loss": 1.48,
+      "step": 870
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "eval_loss": 1.6724848747253418,
+      "eval_runtime": 50.9669,
+      "eval_samples_per_second": 19.621,
+      "eval_steps_per_second": 0.824,
+      "step": 870
+    },
+    {
+      "epoch": 1.0840074673304294,
+      "grad_norm": 2.8183560371398926,
+      "learning_rate": 3.5816420664206646e-06,
+      "loss": 1.4818,
+      "step": 871
+    },
+    {
+      "epoch": 1.0852520224019913,
+      "grad_norm": 2.7531797885894775,
+      "learning_rate": 3.5793357933579336e-06,
+      "loss": 1.4712,
+      "step": 872
+    },
+    {
+      "epoch": 1.086496577473553,
+      "grad_norm": 2.923962354660034,
+      "learning_rate": 3.5770295202952034e-06,
+      "loss": 1.4864,
+      "step": 873
+    },
+    {
+      "epoch": 1.0877411325451152,
+      "grad_norm": 2.851724147796631,
+      "learning_rate": 3.574723247232473e-06,
+      "loss": 1.4771,
+      "step": 874
+    },
+    {
+      "epoch": 1.088985687616677,
+      "grad_norm": 3.024634838104248,
+      "learning_rate": 3.572416974169742e-06,
+      "loss": 1.4643,
+      "step": 875
+    },
+    {
+      "epoch": 1.0902302426882389,
+      "grad_norm": 2.7100844383239746,
+      "learning_rate": 3.5701107011070113e-06,
+      "loss": 1.4773,
+      "step": 876
+    },
+    {
+      "epoch": 1.091474797759801,
+      "grad_norm": 2.730502128601074,
+      "learning_rate": 3.5678044280442807e-06,
+      "loss": 1.5051,
+      "step": 877
+    },
+    {
+      "epoch": 1.0927193528313628,
+      "grad_norm": 2.747082233428955,
+      "learning_rate": 3.5654981549815497e-06,
+      "loss": 1.5207,
+      "step": 878
+    },
+    {
+      "epoch": 1.0939639079029246,
+      "grad_norm": 2.718358039855957,
+      "learning_rate": 3.5631918819188195e-06,
+      "loss": 1.5067,
+      "step": 879
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "grad_norm": 2.7794573307037354,
+      "learning_rate": 3.560885608856089e-06,
+      "loss": 1.4616,
+      "step": 880
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "eval_loss": 1.6676132678985596,
+      "eval_runtime": 51.6157,
+      "eval_samples_per_second": 19.374,
+      "eval_steps_per_second": 0.814,
+      "step": 880
+    },
+    {
+      "epoch": 1.0964530180460486,
+      "grad_norm": 2.727416753768921,
+      "learning_rate": 3.5585793357933584e-06,
+      "loss": 1.4043,
+      "step": 881
+    },
+    {
+      "epoch": 1.0976975731176104,
+      "grad_norm": 2.856900930404663,
+      "learning_rate": 3.5562730627306274e-06,
+      "loss": 1.5198,
+      "step": 882
+    },
+    {
+      "epoch": 1.0989421281891725,
+      "grad_norm": 2.872823715209961,
+      "learning_rate": 3.553966789667897e-06,
+      "loss": 1.4981,
+      "step": 883
+    },
+    {
+      "epoch": 1.1001866832607343,
+      "grad_norm": 2.7724361419677734,
+      "learning_rate": 3.5516605166051667e-06,
+      "loss": 1.4819,
+      "step": 884
+    },
+    {
+      "epoch": 1.1014312383322962,
+      "grad_norm": 2.884434461593628,
+      "learning_rate": 3.5493542435424357e-06,
+      "loss": 1.433,
+      "step": 885
+    },
+    {
+      "epoch": 1.102675793403858,
+      "grad_norm": 2.887873411178589,
+      "learning_rate": 3.547047970479705e-06,
+      "loss": 1.4677,
+      "step": 886
+    },
+    {
+      "epoch": 1.10392034847542,
+      "grad_norm": 2.946845054626465,
+      "learning_rate": 3.5447416974169745e-06,
+      "loss": 1.4983,
+      "step": 887
+    },
+    {
+      "epoch": 1.105164903546982,
+      "grad_norm": 2.752552032470703,
+      "learning_rate": 3.5424354243542435e-06,
+      "loss": 1.4766,
+      "step": 888
+    },
+    {
+      "epoch": 1.1064094586185438,
+      "grad_norm": 2.788634777069092,
+      "learning_rate": 3.5401291512915133e-06,
+      "loss": 1.4747,
+      "step": 889
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "grad_norm": 2.782792568206787,
+      "learning_rate": 3.5378228782287828e-06,
+      "loss": 1.5503,
+      "step": 890
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "eval_loss": 1.6698333024978638,
+      "eval_runtime": 49.1373,
+      "eval_samples_per_second": 20.351,
+      "eval_steps_per_second": 0.855,
+      "step": 890
+    },
+    {
+      "epoch": 1.1088985687616677,
+      "grad_norm": 2.8351166248321533,
+      "learning_rate": 3.535516605166052e-06,
+      "loss": 1.4589,
+      "step": 891
+    },
+    {
+      "epoch": 1.1101431238332296,
+      "grad_norm": 2.798872709274292,
+      "learning_rate": 3.533210332103321e-06,
+      "loss": 1.4496,
+      "step": 892
+    },
+    {
+      "epoch": 1.1113876789047916,
+      "grad_norm": 3.0863239765167236,
+      "learning_rate": 3.5309040590405906e-06,
+      "loss": 1.4698,
+      "step": 893
+    },
+    {
+      "epoch": 1.1126322339763535,
+      "grad_norm": 2.853862762451172,
+      "learning_rate": 3.5285977859778605e-06,
+      "loss": 1.4429,
+      "step": 894
+    },
+    {
+      "epoch": 1.1138767890479153,
+      "grad_norm": 2.8946170806884766,
+      "learning_rate": 3.5262915129151295e-06,
+      "loss": 1.4697,
+      "step": 895
+    },
+    {
+      "epoch": 1.1151213441194774,
+      "grad_norm": 3.043823003768921,
+      "learning_rate": 3.523985239852399e-06,
+      "loss": 1.4666,
+      "step": 896
+    },
+    {
+      "epoch": 1.1163658991910392,
+      "grad_norm": 2.7822980880737305,
+      "learning_rate": 3.5216789667896683e-06,
+      "loss": 1.4523,
+      "step": 897
+    },
+    {
+      "epoch": 1.117610454262601,
+      "grad_norm": 2.9454171657562256,
+      "learning_rate": 3.5193726937269373e-06,
+      "loss": 1.5009,
+      "step": 898
+    },
+    {
+      "epoch": 1.1188550093341632,
+      "grad_norm": 2.8533740043640137,
+      "learning_rate": 3.5170664206642067e-06,
+      "loss": 1.4265,
+      "step": 899
+    },
+    {
+      "epoch": 1.120099564405725,
+      "grad_norm": 2.894937038421631,
+      "learning_rate": 3.5147601476014766e-06,
+      "loss": 1.4895,
+      "step": 900
+    },
+    {
+      "epoch": 1.120099564405725,
+      "eval_loss": 1.6701407432556152,
+      "eval_runtime": 45.477,
+      "eval_samples_per_second": 21.989,
+      "eval_steps_per_second": 0.924,
+      "step": 900
+    },
+    {
+      "epoch": 1.1213441194772868,
+      "grad_norm": 2.933821201324463,
+      "learning_rate": 3.5124538745387456e-06,
+      "loss": 1.468,
+      "step": 901
+    },
+    {
+      "epoch": 1.1225886745488487,
+      "grad_norm": 2.834265947341919,
+      "learning_rate": 3.510147601476015e-06,
+      "loss": 1.4621,
+      "step": 902
+    },
+    {
+      "epoch": 1.1238332296204108,
+      "grad_norm": 2.9064722061157227,
+      "learning_rate": 3.5078413284132844e-06,
+      "loss": 1.504,
+      "step": 903
+    },
+    {
+      "epoch": 1.1250777846919726,
+      "grad_norm": 2.8753886222839355,
+      "learning_rate": 3.505535055350554e-06,
+      "loss": 1.4643,
+      "step": 904
+    },
+    {
+      "epoch": 1.1263223397635345,
+      "grad_norm": 2.894659996032715,
+      "learning_rate": 3.503228782287823e-06,
+      "loss": 1.4909,
+      "step": 905
+    },
+    {
+      "epoch": 1.1275668948350965,
+      "grad_norm": 3.085977792739868,
+      "learning_rate": 3.5009225092250927e-06,
+      "loss": 1.459,
+      "step": 906
+    },
+    {
+      "epoch": 1.1288114499066584,
+      "grad_norm": 2.9958064556121826,
+      "learning_rate": 3.498616236162362e-06,
+      "loss": 1.4152,
+      "step": 907
+    },
+    {
+      "epoch": 1.1300560049782202,
+      "grad_norm": 2.9136762619018555,
+      "learning_rate": 3.496309963099631e-06,
+      "loss": 1.473,
+      "step": 908
+    },
+    {
+      "epoch": 1.1313005600497823,
+      "grad_norm": 2.956437826156616,
+      "learning_rate": 3.4940036900369005e-06,
+      "loss": 1.5167,
+      "step": 909
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "grad_norm": 2.7354865074157715,
+      "learning_rate": 3.4916974169741704e-06,
+      "loss": 1.4413,
+      "step": 910
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "eval_loss": 1.6629912853240967,
+      "eval_runtime": 41.5478,
+      "eval_samples_per_second": 24.069,
+      "eval_steps_per_second": 1.011,
+      "step": 910
+    },
+    {
+      "epoch": 1.133789670192906,
+      "grad_norm": 2.7443008422851562,
+      "learning_rate": 3.4893911439114394e-06,
+      "loss": 1.4475,
+      "step": 911
+    },
+    {
+      "epoch": 1.135034225264468,
+      "grad_norm": 2.7811238765716553,
+      "learning_rate": 3.487084870848709e-06,
+      "loss": 1.4257,
+      "step": 912
+    },
+    {
+      "epoch": 1.13627878033603,
+      "grad_norm": 2.7916479110717773,
+      "learning_rate": 3.4847785977859782e-06,
+      "loss": 1.4048,
+      "step": 913
+    },
+    {
+      "epoch": 1.1375233354075918,
+      "grad_norm": 2.7835633754730225,
+      "learning_rate": 3.4824723247232472e-06,
+      "loss": 1.3996,
+      "step": 914
+    },
+    {
+      "epoch": 1.1387678904791536,
+      "grad_norm": 2.8188705444335938,
+      "learning_rate": 3.4801660516605166e-06,
+      "loss": 1.4541,
+      "step": 915
+    },
+    {
+      "epoch": 1.1400124455507157,
+      "grad_norm": 3.0134100914001465,
+      "learning_rate": 3.4778597785977865e-06,
+      "loss": 1.4353,
+      "step": 916
+    },
+    {
+      "epoch": 1.1412570006222775,
+      "grad_norm": 2.6965606212615967,
+      "learning_rate": 3.475553505535056e-06,
+      "loss": 1.4308,
+      "step": 917
+    },
+    {
+      "epoch": 1.1425015556938394,
+      "grad_norm": 2.8179919719696045,
+      "learning_rate": 3.473247232472325e-06,
+      "loss": 1.5204,
+      "step": 918
+    },
+    {
+      "epoch": 1.1437461107654014,
+      "grad_norm": 2.7213902473449707,
+      "learning_rate": 3.4709409594095943e-06,
+      "loss": 1.4344,
+      "step": 919
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "grad_norm": 2.850193738937378,
+      "learning_rate": 3.4686346863468638e-06,
+      "loss": 1.473,
+      "step": 920
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "eval_loss": 1.663345217704773,
+      "eval_runtime": 43.8397,
+      "eval_samples_per_second": 22.81,
+      "eval_steps_per_second": 0.958,
+      "step": 920
+    },
+    {
+      "epoch": 1.1462352209085251,
+      "grad_norm": 2.780198574066162,
+      "learning_rate": 3.4663284132841328e-06,
+      "loss": 1.4472,
+      "step": 921
+    },
+    {
+      "epoch": 1.1474797759800872,
+      "grad_norm": 2.9373831748962402,
+      "learning_rate": 3.4640221402214026e-06,
+      "loss": 1.426,
+      "step": 922
+    },
+    {
+      "epoch": 1.148724331051649,
+      "grad_norm": 2.6986873149871826,
+      "learning_rate": 3.461715867158672e-06,
+      "loss": 1.3953,
+      "step": 923
+    },
+    {
+      "epoch": 1.149968886123211,
+      "grad_norm": 2.927441120147705,
+      "learning_rate": 3.459409594095941e-06,
+      "loss": 1.4578,
+      "step": 924
+    },
+    {
+      "epoch": 1.151213441194773,
+      "grad_norm": 2.8469648361206055,
+      "learning_rate": 3.4571033210332105e-06,
+      "loss": 1.4313,
+      "step": 925
+    },
+    {
+      "epoch": 1.1524579962663348,
+      "grad_norm": 2.8114013671875,
+      "learning_rate": 3.4547970479704803e-06,
+      "loss": 1.4267,
+      "step": 926
+    },
+    {
+      "epoch": 1.1537025513378967,
+      "grad_norm": 2.977605104446411,
+      "learning_rate": 3.4524907749077493e-06,
+      "loss": 1.4747,
+      "step": 927
+    },
+    {
+      "epoch": 1.1549471064094585,
+      "grad_norm": 2.9327852725982666,
+      "learning_rate": 3.4501845018450187e-06,
+      "loss": 1.4219,
+      "step": 928
+    },
+    {
+      "epoch": 1.1561916614810206,
+      "grad_norm": 2.844038963317871,
+      "learning_rate": 3.447878228782288e-06,
+      "loss": 1.5146,
+      "step": 929
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "grad_norm": 2.9402201175689697,
+      "learning_rate": 3.4455719557195576e-06,
+      "loss": 1.45,
+      "step": 930
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "eval_loss": 1.6620122194290161,
+      "eval_runtime": 46.1748,
+      "eval_samples_per_second": 21.657,
+      "eval_steps_per_second": 0.91,
+      "step": 930
+    },
+    {
+      "epoch": 1.1586807716241443,
+      "grad_norm": 2.842651128768921,
+      "learning_rate": 3.4432656826568266e-06,
+      "loss": 1.4415,
+      "step": 931
+    },
+    {
+      "epoch": 1.1599253266957064,
+      "grad_norm": 2.8909685611724854,
+      "learning_rate": 3.4409594095940964e-06,
+      "loss": 1.4529,
+      "step": 932
+    },
+    {
+      "epoch": 1.1611698817672682,
+      "grad_norm": 2.8025808334350586,
+      "learning_rate": 3.438653136531366e-06,
+      "loss": 1.4552,
+      "step": 933
+    },
+    {
+      "epoch": 1.16241443683883,
+      "grad_norm": 2.6995794773101807,
+      "learning_rate": 3.436346863468635e-06,
+      "loss": 1.4435,
+      "step": 934
+    },
+    {
+      "epoch": 1.1636589919103921,
+      "grad_norm": 2.7307121753692627,
+      "learning_rate": 3.4340405904059043e-06,
+      "loss": 1.4447,
+      "step": 935
+    },
+    {
+      "epoch": 1.164903546981954,
+      "grad_norm": 2.9532501697540283,
+      "learning_rate": 3.4317343173431737e-06,
+      "loss": 1.4747,
+      "step": 936
+    },
+    {
+      "epoch": 1.1661481020535158,
+      "grad_norm": 3.0135788917541504,
+      "learning_rate": 3.4294280442804427e-06,
+      "loss": 1.525,
+      "step": 937
+    },
+    {
+      "epoch": 1.167392657125078,
+      "grad_norm": 2.796355962753296,
+      "learning_rate": 3.4271217712177125e-06,
+      "loss": 1.4539,
+      "step": 938
+    },
+    {
+      "epoch": 1.1686372121966397,
+      "grad_norm": 2.9422309398651123,
+      "learning_rate": 3.424815498154982e-06,
+      "loss": 1.4664,
+      "step": 939
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "grad_norm": 2.7680039405822754,
+      "learning_rate": 3.4225092250922514e-06,
+      "loss": 1.4837,
+      "step": 940
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "eval_loss": 1.660908818244934,
+      "eval_runtime": 45.6998,
+      "eval_samples_per_second": 21.882,
+      "eval_steps_per_second": 0.919,
+      "step": 940
+    },
+    {
+      "epoch": 1.1711263223397634,
+      "grad_norm": 2.8888845443725586,
+      "learning_rate": 3.4202029520295204e-06,
+      "loss": 1.5315,
+      "step": 941
+    },
+    {
+      "epoch": 1.1723708774113255,
+      "grad_norm": 2.7124130725860596,
+      "learning_rate": 3.41789667896679e-06,
+      "loss": 1.4188,
+      "step": 942
+    },
+    {
+      "epoch": 1.1736154324828874,
+      "grad_norm": 2.75203013420105,
+      "learning_rate": 3.4155904059040596e-06,
+      "loss": 1.4533,
+      "step": 943
+    },
+    {
+      "epoch": 1.1748599875544492,
+      "grad_norm": 2.865480422973633,
+      "learning_rate": 3.4132841328413286e-06,
+      "loss": 1.4532,
+      "step": 944
+    },
+    {
+      "epoch": 1.1761045426260113,
+      "grad_norm": 2.8618321418762207,
+      "learning_rate": 3.410977859778598e-06,
+      "loss": 1.4386,
+      "step": 945
+    },
+    {
+      "epoch": 1.1773490976975731,
+      "grad_norm": 2.7656145095825195,
+      "learning_rate": 3.4086715867158675e-06,
+      "loss": 1.3996,
+      "step": 946
+    },
+    {
+      "epoch": 1.178593652769135,
+      "grad_norm": 2.76213002204895,
+      "learning_rate": 3.4063653136531365e-06,
+      "loss": 1.4979,
+      "step": 947
+    },
+    {
+      "epoch": 1.179838207840697,
+      "grad_norm": 2.8581957817077637,
+      "learning_rate": 3.4040590405904063e-06,
+      "loss": 1.4673,
+      "step": 948
+    },
+    {
+      "epoch": 1.1810827629122589,
+      "grad_norm": 2.7825586795806885,
+      "learning_rate": 3.4017527675276758e-06,
+      "loss": 1.4453,
+      "step": 949
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "grad_norm": 2.9226131439208984,
+      "learning_rate": 3.3994464944649448e-06,
+      "loss": 1.4879,
+      "step": 950
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "eval_loss": 1.6621263027191162,
+      "eval_runtime": 45.1222,
+      "eval_samples_per_second": 22.162,
+      "eval_steps_per_second": 0.931,
+      "step": 950
+    },
+    {
+      "epoch": 1.1835718730553828,
+      "grad_norm": 2.9997975826263428,
+      "learning_rate": 3.397140221402214e-06,
+      "loss": 1.505,
+      "step": 951
+    },
+    {
+      "epoch": 1.1848164281269447,
+      "grad_norm": 2.7225453853607178,
+      "learning_rate": 3.3948339483394836e-06,
+      "loss": 1.4695,
+      "step": 952
+    },
+    {
+      "epoch": 1.1860609831985065,
+      "grad_norm": 2.916473150253296,
+      "learning_rate": 3.3925276752767534e-06,
+      "loss": 1.4901,
+      "step": 953
+    },
+    {
+      "epoch": 1.1873055382700684,
+      "grad_norm": 2.8371644020080566,
+      "learning_rate": 3.3902214022140224e-06,
+      "loss": 1.4944,
+      "step": 954
+    },
+    {
+      "epoch": 1.1885500933416304,
+      "grad_norm": 2.813662528991699,
+      "learning_rate": 3.387915129151292e-06,
+      "loss": 1.4603,
+      "step": 955
+    },
+    {
+      "epoch": 1.1897946484131923,
+      "grad_norm": 2.991931676864624,
+      "learning_rate": 3.3856088560885613e-06,
+      "loss": 1.5496,
+      "step": 956
+    },
+    {
+      "epoch": 1.1910392034847541,
+      "grad_norm": 2.6252501010894775,
+      "learning_rate": 3.3833025830258303e-06,
+      "loss": 1.4405,
+      "step": 957
+    },
+    {
+      "epoch": 1.1922837585563162,
+      "grad_norm": 2.747725009918213,
+      "learning_rate": 3.3809963099630997e-06,
+      "loss": 1.4689,
+      "step": 958
+    },
+    {
+      "epoch": 1.193528313627878,
+      "grad_norm": 2.887763261795044,
+      "learning_rate": 3.3786900369003696e-06,
+      "loss": 1.4501,
+      "step": 959
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "grad_norm": 2.9954354763031006,
+      "learning_rate": 3.3763837638376386e-06,
+      "loss": 1.4978,
+      "step": 960
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "eval_loss": 1.664442539215088,
+      "eval_runtime": 49.1266,
+      "eval_samples_per_second": 20.356,
+      "eval_steps_per_second": 0.855,
+      "step": 960
+    },
+    {
+      "epoch": 1.196017423771002,
+      "grad_norm": 3.0174152851104736,
+      "learning_rate": 3.374077490774908e-06,
+      "loss": 1.4353,
+      "step": 961
+    },
+    {
+      "epoch": 1.1972619788425638,
+      "grad_norm": 2.9082882404327393,
+      "learning_rate": 3.3717712177121774e-06,
+      "loss": 1.4878,
+      "step": 962
+    },
+    {
+      "epoch": 1.1985065339141256,
+      "grad_norm": 2.897993326187134,
+      "learning_rate": 3.3694649446494464e-06,
+      "loss": 1.4975,
+      "step": 963
+    },
+    {
+      "epoch": 1.1997510889856877,
+      "grad_norm": 2.739470958709717,
+      "learning_rate": 3.3671586715867163e-06,
+      "loss": 1.4202,
+      "step": 964
+    },
+    {
+      "epoch": 1.2009956440572496,
+      "grad_norm": 2.817786693572998,
+      "learning_rate": 3.3648523985239857e-06,
+      "loss": 1.5195,
+      "step": 965
+    },
+    {
+      "epoch": 1.2022401991288114,
+      "grad_norm": 2.8476386070251465,
+      "learning_rate": 3.362546125461255e-06,
+      "loss": 1.444,
+      "step": 966
+    },
+    {
+      "epoch": 1.2034847542003733,
+      "grad_norm": 2.804047107696533,
+      "learning_rate": 3.360239852398524e-06,
+      "loss": 1.4442,
+      "step": 967
+    },
+    {
+      "epoch": 1.2047293092719353,
+      "grad_norm": 2.9285430908203125,
+      "learning_rate": 3.3579335793357935e-06,
+      "loss": 1.4877,
+      "step": 968
+    },
+    {
+      "epoch": 1.2059738643434972,
+      "grad_norm": 2.8814878463745117,
+      "learning_rate": 3.3556273062730634e-06,
+      "loss": 1.5078,
+      "step": 969
+    },
+    {
+      "epoch": 1.207218419415059,
+      "grad_norm": 2.970728635787964,
+      "learning_rate": 3.3533210332103324e-06,
+      "loss": 1.4166,
+      "step": 970
+    },
+    {
+      "epoch": 1.207218419415059,
+      "eval_loss": 1.6635315418243408,
+      "eval_runtime": 47.2321,
+      "eval_samples_per_second": 21.172,
+      "eval_steps_per_second": 0.889,
+      "step": 970
+    },
+    {
+      "epoch": 1.208462974486621,
+      "grad_norm": 2.9836292266845703,
+      "learning_rate": 3.3510147601476018e-06,
+      "loss": 1.4777,
+      "step": 971
+    },
+    {
+      "epoch": 1.209707529558183,
+      "grad_norm": 2.8495216369628906,
+      "learning_rate": 3.348708487084871e-06,
+      "loss": 1.4829,
+      "step": 972
+    },
+    {
+      "epoch": 1.2109520846297448,
+      "grad_norm": 2.7709312438964844,
+      "learning_rate": 3.34640221402214e-06,
+      "loss": 1.4242,
+      "step": 973
+    },
+    {
+      "epoch": 1.2121966397013069,
+      "grad_norm": 2.999462366104126,
+      "learning_rate": 3.3440959409594096e-06,
+      "loss": 1.4148,
+      "step": 974
+    },
+    {
+      "epoch": 1.2134411947728687,
+      "grad_norm": 2.8719074726104736,
+      "learning_rate": 3.3417896678966795e-06,
+      "loss": 1.4196,
+      "step": 975
+    },
+    {
+      "epoch": 1.2146857498444306,
+      "grad_norm": 2.8469338417053223,
+      "learning_rate": 3.3394833948339485e-06,
+      "loss": 1.4941,
+      "step": 976
+    },
+    {
+      "epoch": 1.2159303049159926,
+      "grad_norm": 2.9780938625335693,
+      "learning_rate": 3.337177121771218e-06,
+      "loss": 1.513,
+      "step": 977
+    },
+    {
+      "epoch": 1.2171748599875545,
+      "grad_norm": 2.814760684967041,
+      "learning_rate": 3.3348708487084873e-06,
+      "loss": 1.4204,
+      "step": 978
+    },
+    {
+      "epoch": 1.2184194150591163,
+      "grad_norm": 3.016261339187622,
+      "learning_rate": 3.332564575645757e-06,
+      "loss": 1.4389,
+      "step": 979
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "grad_norm": 2.9385855197906494,
+      "learning_rate": 3.330258302583026e-06,
+      "loss": 1.4885,
+      "step": 980
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "eval_loss": 1.6688873767852783,
+      "eval_runtime": 46.2942,
+      "eval_samples_per_second": 21.601,
+      "eval_steps_per_second": 0.907,
+      "step": 980
+    },
+    {
+      "epoch": 1.2209085252022402,
+      "grad_norm": 3.1434757709503174,
+      "learning_rate": 3.3279520295202956e-06,
+      "loss": 1.5157,
+      "step": 981
+    },
+    {
+      "epoch": 1.222153080273802,
+      "grad_norm": 3.0104827880859375,
+      "learning_rate": 3.325645756457565e-06,
+      "loss": 1.49,
+      "step": 982
+    },
+    {
+      "epoch": 1.223397635345364,
+      "grad_norm": 3.1913645267486572,
+      "learning_rate": 3.323339483394834e-06,
+      "loss": 1.4937,
+      "step": 983
+    },
+    {
+      "epoch": 1.224642190416926,
+      "grad_norm": 3.19496750831604,
+      "learning_rate": 3.3210332103321034e-06,
+      "loss": 1.5197,
+      "step": 984
+    },
+    {
+      "epoch": 1.2258867454884879,
+      "grad_norm": 3.046232223510742,
+      "learning_rate": 3.3187269372693733e-06,
+      "loss": 1.4497,
+      "step": 985
+    },
+    {
+      "epoch": 1.2271313005600497,
+      "grad_norm": 2.850675344467163,
+      "learning_rate": 3.3164206642066423e-06,
+      "loss": 1.4595,
+      "step": 986
+    },
+    {
+      "epoch": 1.2283758556316118,
+      "grad_norm": 2.7744576930999756,
+      "learning_rate": 3.3141143911439117e-06,
+      "loss": 1.4726,
+      "step": 987
+    },
+    {
+      "epoch": 1.2296204107031736,
+      "grad_norm": 3.017153024673462,
+      "learning_rate": 3.311808118081181e-06,
+      "loss": 1.493,
+      "step": 988
+    },
+    {
+      "epoch": 1.2308649657747355,
+      "grad_norm": 2.8821120262145996,
+      "learning_rate": 3.3095018450184506e-06,
+      "loss": 1.4734,
+      "step": 989
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "grad_norm": 2.8805840015411377,
+      "learning_rate": 3.3071955719557196e-06,
+      "loss": 1.4562,
+      "step": 990
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "eval_loss": 1.669345736503601,
+      "eval_runtime": 47.0893,
+      "eval_samples_per_second": 21.236,
+      "eval_steps_per_second": 0.892,
+      "step": 990
+    },
+    {
+      "epoch": 1.2333540759178594,
+      "grad_norm": 2.7599008083343506,
+      "learning_rate": 3.3048892988929894e-06,
+      "loss": 1.4218,
+      "step": 991
+    },
+    {
+      "epoch": 1.2345986309894212,
+      "grad_norm": 2.9835240840911865,
+      "learning_rate": 3.302583025830259e-06,
+      "loss": 1.4596,
+      "step": 992
+    },
+    {
+      "epoch": 1.235843186060983,
+      "grad_norm": 2.843358039855957,
+      "learning_rate": 3.300276752767528e-06,
+      "loss": 1.4561,
+      "step": 993
+    },
+    {
+      "epoch": 1.2370877411325452,
+      "grad_norm": 2.7386093139648438,
+      "learning_rate": 3.2979704797047972e-06,
+      "loss": 1.4475,
+      "step": 994
+    },
+    {
+      "epoch": 1.238332296204107,
+      "grad_norm": 2.668161392211914,
+      "learning_rate": 3.2956642066420667e-06,
+      "loss": 1.4661,
+      "step": 995
+    },
+    {
+      "epoch": 1.2395768512756689,
+      "grad_norm": 2.947235107421875,
+      "learning_rate": 3.2933579335793357e-06,
+      "loss": 1.4358,
+      "step": 996
+    },
+    {
+      "epoch": 1.240821406347231,
+      "grad_norm": 2.8159878253936768,
+      "learning_rate": 3.2910516605166055e-06,
+      "loss": 1.4432,
+      "step": 997
+    },
+    {
+      "epoch": 1.2420659614187928,
+      "grad_norm": 2.896597385406494,
+      "learning_rate": 3.288745387453875e-06,
+      "loss": 1.5585,
+      "step": 998
+    },
+    {
+      "epoch": 1.2433105164903546,
+      "grad_norm": 2.80241322517395,
+      "learning_rate": 3.286439114391144e-06,
+      "loss": 1.5076,
+      "step": 999
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "grad_norm": 3.008939027786255,
+      "learning_rate": 3.2841328413284134e-06,
+      "loss": 1.4234,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "eval_loss": 1.6621588468551636,
+      "eval_runtime": 50.4926,
+      "eval_samples_per_second": 19.805,
+      "eval_steps_per_second": 0.832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2457996266334785,
+      "grad_norm": 2.9872381687164307,
+      "learning_rate": 3.281826568265683e-06,
+      "loss": 1.453,
+      "step": 1001
+    },
+    {
+      "epoch": 1.2470441817050404,
+      "grad_norm": 2.704676628112793,
+      "learning_rate": 3.2795202952029526e-06,
+      "loss": 1.4711,
+      "step": 1002
+    },
+    {
+      "epoch": 1.2482887367766025,
+      "grad_norm": 2.8100757598876953,
+      "learning_rate": 3.2772140221402216e-06,
+      "loss": 1.3884,
+      "step": 1003
+    },
+    {
+      "epoch": 1.2495332918481643,
+      "grad_norm": 2.902953624725342,
+      "learning_rate": 3.274907749077491e-06,
+      "loss": 1.4986,
+      "step": 1004
+    },
+    {
+      "epoch": 1.2507778469197262,
+      "grad_norm": 2.980163097381592,
+      "learning_rate": 3.2726014760147605e-06,
+      "loss": 1.5067,
+      "step": 1005
+    },
+    {
+      "epoch": 1.252022401991288,
+      "grad_norm": 2.8820345401763916,
+      "learning_rate": 3.2702952029520295e-06,
+      "loss": 1.4995,
+      "step": 1006
+    },
+    {
+      "epoch": 1.25326695706285,
+      "grad_norm": 2.8219635486602783,
+      "learning_rate": 3.2679889298892993e-06,
+      "loss": 1.4881,
+      "step": 1007
+    },
+    {
+      "epoch": 1.254511512134412,
+      "grad_norm": 2.8593199253082275,
+      "learning_rate": 3.2656826568265687e-06,
+      "loss": 1.4469,
+      "step": 1008
+    },
+    {
+      "epoch": 1.255756067205974,
+      "grad_norm": 2.888357639312744,
+      "learning_rate": 3.2633763837638377e-06,
+      "loss": 1.4869,
+      "step": 1009
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "grad_norm": 2.9985013008117676,
+      "learning_rate": 3.261070110701107e-06,
+      "loss": 1.4724,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "eval_loss": 1.6492141485214233,
+      "eval_runtime": 45.2197,
+      "eval_samples_per_second": 22.114,
+      "eval_steps_per_second": 0.929,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2582451773490977,
+      "grad_norm": 2.860780715942383,
+      "learning_rate": 3.2587638376383766e-06,
+      "loss": 1.4618,
+      "step": 1011
+    },
+    {
+      "epoch": 1.2594897324206595,
+      "grad_norm": 2.766479730606079,
+      "learning_rate": 3.2564575645756456e-06,
+      "loss": 1.4642,
+      "step": 1012
+    },
+    {
+      "epoch": 1.2607342874922216,
+      "grad_norm": 2.711759328842163,
+      "learning_rate": 3.2541512915129154e-06,
+      "loss": 1.4283,
+      "step": 1013
+    },
+    {
+      "epoch": 1.2619788425637835,
+      "grad_norm": 2.8365769386291504,
+      "learning_rate": 3.251845018450185e-06,
+      "loss": 1.4248,
+      "step": 1014
+    },
+    {
+      "epoch": 1.2632233976353453,
+      "grad_norm": 2.957620143890381,
+      "learning_rate": 3.2495387453874543e-06,
+      "loss": 1.4662,
+      "step": 1015
+    },
+    {
+      "epoch": 1.2644679527069074,
+      "grad_norm": 2.8066208362579346,
+      "learning_rate": 3.2472324723247233e-06,
+      "loss": 1.4447,
+      "step": 1016
+    },
+    {
+      "epoch": 1.2657125077784692,
+      "grad_norm": 2.902040958404541,
+      "learning_rate": 3.244926199261993e-06,
+      "loss": 1.4743,
+      "step": 1017
+    },
+    {
+      "epoch": 1.266957062850031,
+      "grad_norm": 2.7636687755584717,
+      "learning_rate": 3.2426199261992625e-06,
+      "loss": 1.4465,
+      "step": 1018
+    },
+    {
+      "epoch": 1.268201617921593,
+      "grad_norm": 2.95261549949646,
+      "learning_rate": 3.2403136531365315e-06,
+      "loss": 1.4769,
+      "step": 1019
+    },
+    {
+      "epoch": 1.269446172993155,
+      "grad_norm": 2.911128520965576,
+      "learning_rate": 3.238007380073801e-06,
+      "loss": 1.3843,
+      "step": 1020
+    },
+    {
+      "epoch": 1.269446172993155,
+      "eval_loss": 1.6526964902877808,
+      "eval_runtime": 48.2126,
+      "eval_samples_per_second": 20.741,
+      "eval_steps_per_second": 0.871,
+      "step": 1020
+    },
+    {
+      "epoch": 1.2706907280647168,
+      "grad_norm": 2.769951820373535,
+      "learning_rate": 3.2357011070110704e-06,
+      "loss": 1.4847,
+      "step": 1021
+    },
+    {
+      "epoch": 1.271935283136279,
+      "grad_norm": 3.0451674461364746,
+      "learning_rate": 3.2333948339483394e-06,
+      "loss": 1.4453,
+      "step": 1022
+    },
+    {
+      "epoch": 1.2731798382078408,
+      "grad_norm": 2.9543521404266357,
+      "learning_rate": 3.2310885608856092e-06,
+      "loss": 1.4789,
+      "step": 1023
+    },
+    {
+      "epoch": 1.2744243932794026,
+      "grad_norm": 2.9174394607543945,
+      "learning_rate": 3.2287822878228787e-06,
+      "loss": 1.4807,
+      "step": 1024
+    },
+    {
+      "epoch": 1.2756689483509644,
+      "grad_norm": 2.888258934020996,
+      "learning_rate": 3.2264760147601477e-06,
+      "loss": 1.4367,
+      "step": 1025
+    },
+    {
+      "epoch": 1.2769135034225265,
+      "grad_norm": 2.8346047401428223,
+      "learning_rate": 3.224169741697417e-06,
+      "loss": 1.4313,
+      "step": 1026
+    },
+    {
+      "epoch": 1.2781580584940884,
+      "grad_norm": 2.7531332969665527,
+      "learning_rate": 3.2218634686346865e-06,
+      "loss": 1.4018,
+      "step": 1027
+    },
+    {
+      "epoch": 1.2794026135656502,
+      "grad_norm": 3.136946439743042,
+      "learning_rate": 3.2195571955719564e-06,
+      "loss": 1.4417,
+      "step": 1028
+    },
+    {
+      "epoch": 1.2806471686372123,
+      "grad_norm": 2.9618165493011475,
+      "learning_rate": 3.2172509225092254e-06,
+      "loss": 1.5117,
+      "step": 1029
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "grad_norm": 3.0740747451782227,
+      "learning_rate": 3.2149446494464948e-06,
+      "loss": 1.4641,
+      "step": 1030
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "eval_loss": 1.6572761535644531,
+      "eval_runtime": 51.6696,
+      "eval_samples_per_second": 19.354,
+      "eval_steps_per_second": 0.813,
+      "step": 1030
+    },
+    {
+      "epoch": 1.283136278780336,
+      "grad_norm": 2.9225106239318848,
+      "learning_rate": 3.212638376383764e-06,
+      "loss": 1.4421,
+      "step": 1031
+    },
+    {
+      "epoch": 1.2843808338518978,
+      "grad_norm": 2.9530649185180664,
+      "learning_rate": 3.210332103321033e-06,
+      "loss": 1.5138,
+      "step": 1032
+    },
+    {
+      "epoch": 1.28562538892346,
+      "grad_norm": 2.8513238430023193,
+      "learning_rate": 3.208025830258303e-06,
+      "loss": 1.411,
+      "step": 1033
+    },
+    {
+      "epoch": 1.2868699439950217,
+      "grad_norm": 2.7440171241760254,
+      "learning_rate": 3.2057195571955725e-06,
+      "loss": 1.4182,
+      "step": 1034
+    },
+    {
+      "epoch": 1.2881144990665838,
+      "grad_norm": 2.8126630783081055,
+      "learning_rate": 3.2034132841328415e-06,
+      "loss": 1.4333,
+      "step": 1035
+    },
+    {
+      "epoch": 1.2893590541381457,
+      "grad_norm": 2.8598744869232178,
+      "learning_rate": 3.201107011070111e-06,
+      "loss": 1.439,
+      "step": 1036
+    },
+    {
+      "epoch": 1.2906036092097075,
+      "grad_norm": 2.901622772216797,
+      "learning_rate": 3.1988007380073803e-06,
+      "loss": 1.4461,
+      "step": 1037
+    },
+    {
+      "epoch": 1.2918481642812694,
+      "grad_norm": 2.7670180797576904,
+      "learning_rate": 3.1964944649446493e-06,
+      "loss": 1.4772,
+      "step": 1038
+    },
+    {
+      "epoch": 1.2930927193528314,
+      "grad_norm": 2.7860350608825684,
+      "learning_rate": 3.194188191881919e-06,
+      "loss": 1.4414,
+      "step": 1039
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "grad_norm": 2.822052478790283,
+      "learning_rate": 3.1918819188191886e-06,
+      "loss": 1.4192,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "eval_loss": 1.663284420967102,
+      "eval_runtime": 47.2822,
+      "eval_samples_per_second": 21.15,
+      "eval_steps_per_second": 0.888,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2955818294959551,
+      "grad_norm": 2.909109115600586,
+      "learning_rate": 3.189575645756458e-06,
+      "loss": 1.4795,
+      "step": 1041
+    },
+    {
+      "epoch": 1.2968263845675172,
+      "grad_norm": 2.933673143386841,
+      "learning_rate": 3.187269372693727e-06,
+      "loss": 1.5038,
+      "step": 1042
+    },
+    {
+      "epoch": 1.298070939639079,
+      "grad_norm": 3.032518148422241,
+      "learning_rate": 3.1849630996309964e-06,
+      "loss": 1.5133,
+      "step": 1043
+    },
+    {
+      "epoch": 1.299315494710641,
+      "grad_norm": 2.826040506362915,
+      "learning_rate": 3.1826568265682663e-06,
+      "loss": 1.4395,
+      "step": 1044
+    },
+    {
+      "epoch": 1.3005600497822027,
+      "grad_norm": 2.8152658939361572,
+      "learning_rate": 3.1803505535055353e-06,
+      "loss": 1.4787,
+      "step": 1045
+    },
+    {
+      "epoch": 1.3018046048537648,
+      "grad_norm": 2.9410431385040283,
+      "learning_rate": 3.1780442804428047e-06,
+      "loss": 1.4375,
+      "step": 1046
+    },
+    {
+      "epoch": 1.3030491599253267,
+      "grad_norm": 2.948636531829834,
+      "learning_rate": 3.175738007380074e-06,
+      "loss": 1.4847,
+      "step": 1047
+    },
+    {
+      "epoch": 1.3042937149968887,
+      "grad_norm": 2.6287615299224854,
+      "learning_rate": 3.173431734317343e-06,
+      "loss": 1.4186,
+      "step": 1048
+    },
+    {
+      "epoch": 1.3055382700684506,
+      "grad_norm": 2.836029052734375,
+      "learning_rate": 3.1711254612546125e-06,
+      "loss": 1.4251,
+      "step": 1049
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "grad_norm": 2.982081651687622,
+      "learning_rate": 3.1688191881918824e-06,
+      "loss": 1.4951,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "eval_loss": 1.6578067541122437,
+      "eval_runtime": 47.5847,
+      "eval_samples_per_second": 21.015,
+      "eval_steps_per_second": 0.883,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3080273802115743,
+      "grad_norm": 2.9297118186950684,
+      "learning_rate": 3.166512915129152e-06,
+      "loss": 1.4699,
+      "step": 1051
+    },
+    {
+      "epoch": 1.3092719352831363,
+      "grad_norm": 2.906435251235962,
+      "learning_rate": 3.164206642066421e-06,
+      "loss": 1.4562,
+      "step": 1052
+    },
+    {
+      "epoch": 1.3105164903546982,
+      "grad_norm": 2.8655896186828613,
+      "learning_rate": 3.1619003690036902e-06,
+      "loss": 1.4351,
+      "step": 1053
+    },
+    {
+      "epoch": 1.31176104542626,
+      "grad_norm": 2.779244899749756,
+      "learning_rate": 3.15959409594096e-06,
+      "loss": 1.4479,
+      "step": 1054
+    },
+    {
+      "epoch": 1.3130056004978221,
+      "grad_norm": 2.9863739013671875,
+      "learning_rate": 3.157287822878229e-06,
+      "loss": 1.4964,
+      "step": 1055
+    },
+    {
+      "epoch": 1.314250155569384,
+      "grad_norm": 2.821871042251587,
+      "learning_rate": 3.1549815498154985e-06,
+      "loss": 1.4173,
+      "step": 1056
+    },
+    {
+      "epoch": 1.3154947106409458,
+      "grad_norm": 2.8573882579803467,
+      "learning_rate": 3.152675276752768e-06,
+      "loss": 1.4877,
+      "step": 1057
+    },
+    {
+      "epoch": 1.3167392657125077,
+      "grad_norm": 2.9824137687683105,
+      "learning_rate": 3.150369003690037e-06,
+      "loss": 1.5124,
+      "step": 1058
+    },
+    {
+      "epoch": 1.3179838207840697,
+      "grad_norm": 2.881098747253418,
+      "learning_rate": 3.1480627306273063e-06,
+      "loss": 1.4346,
+      "step": 1059
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "grad_norm": 2.918782949447632,
+      "learning_rate": 3.145756457564576e-06,
+      "loss": 1.4797,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "eval_loss": 1.6510179042816162,
+      "eval_runtime": 53.6356,
+      "eval_samples_per_second": 18.644,
+      "eval_steps_per_second": 0.783,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3204729309271936,
+      "grad_norm": 3.045776128768921,
+      "learning_rate": 3.143450184501845e-06,
+      "loss": 1.5039,
+      "step": 1061
+    },
+    {
+      "epoch": 1.3217174859987555,
+      "grad_norm": 3.041501760482788,
+      "learning_rate": 3.1411439114391146e-06,
+      "loss": 1.4878,
+      "step": 1062
+    },
+    {
+      "epoch": 1.3229620410703173,
+      "grad_norm": 2.917837142944336,
+      "learning_rate": 3.138837638376384e-06,
+      "loss": 1.4202,
+      "step": 1063
+    },
+    {
+      "epoch": 1.3242065961418792,
+      "grad_norm": 2.708070755004883,
+      "learning_rate": 3.1365313653136535e-06,
+      "loss": 1.4071,
+      "step": 1064
+    },
+    {
+      "epoch": 1.3254511512134413,
+      "grad_norm": 2.9146156311035156,
+      "learning_rate": 3.1342250922509225e-06,
+      "loss": 1.4428,
+      "step": 1065
+    },
+    {
+      "epoch": 1.326695706285003,
+      "grad_norm": 3.220919132232666,
+      "learning_rate": 3.1319188191881923e-06,
+      "loss": 1.4733,
+      "step": 1066
+    },
+    {
+      "epoch": 1.327940261356565,
+      "grad_norm": 2.9127554893493652,
+      "learning_rate": 3.1296125461254617e-06,
+      "loss": 1.4554,
+      "step": 1067
+    },
+    {
+      "epoch": 1.329184816428127,
+      "grad_norm": 3.053191661834717,
+      "learning_rate": 3.1273062730627307e-06,
+      "loss": 1.4044,
+      "step": 1068
+    },
+    {
+      "epoch": 1.3304293714996889,
+      "grad_norm": 3.2937021255493164,
+      "learning_rate": 3.125e-06,
+      "loss": 1.4949,
+      "step": 1069
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "grad_norm": 2.976717948913574,
+      "learning_rate": 3.12269372693727e-06,
+      "loss": 1.446,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "eval_loss": 1.6503105163574219,
+      "eval_runtime": 50.7579,
+      "eval_samples_per_second": 19.701,
+      "eval_steps_per_second": 0.827,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3329184816428126,
+      "grad_norm": 3.0860936641693115,
+      "learning_rate": 3.120387453874539e-06,
+      "loss": 1.4055,
+      "step": 1071
+    },
+    {
+      "epoch": 1.3341630367143746,
+      "grad_norm": 3.0401217937469482,
+      "learning_rate": 3.1180811808118084e-06,
+      "loss": 1.4688,
+      "step": 1072
+    },
+    {
+      "epoch": 1.3354075917859365,
+      "grad_norm": 2.953253984451294,
+      "learning_rate": 3.115774907749078e-06,
+      "loss": 1.5001,
+      "step": 1073
+    },
+    {
+      "epoch": 1.3366521468574986,
+      "grad_norm": 3.0665059089660645,
+      "learning_rate": 3.113468634686347e-06,
+      "loss": 1.4217,
+      "step": 1074
+    },
+    {
+      "epoch": 1.3378967019290604,
+      "grad_norm": 3.121600389480591,
+      "learning_rate": 3.1111623616236163e-06,
+      "loss": 1.4508,
+      "step": 1075
+    },
+    {
+      "epoch": 1.3391412570006223,
+      "grad_norm": 2.8057339191436768,
+      "learning_rate": 3.108856088560886e-06,
+      "loss": 1.4241,
+      "step": 1076
+    },
+    {
+      "epoch": 1.340385812072184,
+      "grad_norm": 3.0147411823272705,
+      "learning_rate": 3.1065498154981555e-06,
+      "loss": 1.479,
+      "step": 1077
+    },
+    {
+      "epoch": 1.3416303671437462,
+      "grad_norm": 3.030010223388672,
+      "learning_rate": 3.1042435424354245e-06,
+      "loss": 1.4861,
+      "step": 1078
+    },
+    {
+      "epoch": 1.342874922215308,
+      "grad_norm": 3.027998208999634,
+      "learning_rate": 3.101937269372694e-06,
+      "loss": 1.4916,
+      "step": 1079
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "grad_norm": 2.9252536296844482,
+      "learning_rate": 3.0996309963099634e-06,
+      "loss": 1.418,
+      "step": 1080
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "eval_loss": 1.6490333080291748,
+      "eval_runtime": 47.4026,
+      "eval_samples_per_second": 21.096,
+      "eval_steps_per_second": 0.886,
+      "step": 1080
+    },
+    {
+      "epoch": 1.345364032358432,
+      "grad_norm": 3.110847234725952,
+      "learning_rate": 3.0973247232472324e-06,
+      "loss": 1.4127,
+      "step": 1081
+    },
+    {
+      "epoch": 1.3466085874299938,
+      "grad_norm": 2.846491575241089,
+      "learning_rate": 3.0950184501845022e-06,
+      "loss": 1.444,
+      "step": 1082
+    },
+    {
+      "epoch": 1.3478531425015556,
+      "grad_norm": 2.950533151626587,
+      "learning_rate": 3.0927121771217716e-06,
+      "loss": 1.4456,
+      "step": 1083
+    },
+    {
+      "epoch": 1.3490976975731175,
+      "grad_norm": 2.875964403152466,
+      "learning_rate": 3.0904059040590406e-06,
+      "loss": 1.4485,
+      "step": 1084
+    },
+    {
+      "epoch": 1.3503422526446796,
+      "grad_norm": 2.9612326622009277,
+      "learning_rate": 3.08809963099631e-06,
+      "loss": 1.4446,
+      "step": 1085
+    },
+    {
+      "epoch": 1.3515868077162414,
+      "grad_norm": 2.925297975540161,
+      "learning_rate": 3.0857933579335795e-06,
+      "loss": 1.4847,
+      "step": 1086
+    },
+    {
+      "epoch": 1.3528313627878035,
+      "grad_norm": 2.991616725921631,
+      "learning_rate": 3.0834870848708485e-06,
+      "loss": 1.5151,
+      "step": 1087
+    },
+    {
+      "epoch": 1.3540759178593653,
+      "grad_norm": 2.96329927444458,
+      "learning_rate": 3.0811808118081183e-06,
+      "loss": 1.4574,
+      "step": 1088
+    },
+    {
+      "epoch": 1.3553204729309272,
+      "grad_norm": 2.948702573776245,
+      "learning_rate": 3.0788745387453878e-06,
+      "loss": 1.4585,
+      "step": 1089
+    },
+    {
+      "epoch": 1.356565028002489,
+      "grad_norm": 3.218463659286499,
+      "learning_rate": 3.076568265682657e-06,
+      "loss": 1.4439,
+      "step": 1090
+    },
+    {
+      "epoch": 1.356565028002489,
+      "eval_loss": 1.645838975906372,
+      "eval_runtime": 47.048,
+      "eval_samples_per_second": 21.255,
+      "eval_steps_per_second": 0.893,
+      "step": 1090
+    },
+    {
+      "epoch": 1.357809583074051,
+      "grad_norm": 2.9799578189849854,
+      "learning_rate": 3.074261992619926e-06,
+      "loss": 1.4883,
+      "step": 1091
+    },
+    {
+      "epoch": 1.359054138145613,
+      "grad_norm": 2.7660505771636963,
+      "learning_rate": 3.071955719557196e-06,
+      "loss": 1.4349,
+      "step": 1092
+    },
+    {
+      "epoch": 1.3602986932171748,
+      "grad_norm": 3.0201916694641113,
+      "learning_rate": 3.0696494464944655e-06,
+      "loss": 1.418,
+      "step": 1093
+    },
+    {
+      "epoch": 1.3615432482887369,
+      "grad_norm": 2.844207763671875,
+      "learning_rate": 3.0673431734317345e-06,
+      "loss": 1.4769,
+      "step": 1094
+    },
+    {
+      "epoch": 1.3627878033602987,
+      "grad_norm": 2.8306753635406494,
+      "learning_rate": 3.065036900369004e-06,
+      "loss": 1.4653,
+      "step": 1095
+    },
+    {
+      "epoch": 1.3640323584318605,
+      "grad_norm": 2.805023431777954,
+      "learning_rate": 3.0627306273062733e-06,
+      "loss": 1.4771,
+      "step": 1096
+    },
+    {
+      "epoch": 1.3652769135034224,
+      "grad_norm": 3.020883083343506,
+      "learning_rate": 3.0604243542435423e-06,
+      "loss": 1.4237,
+      "step": 1097
+    },
+    {
+      "epoch": 1.3665214685749845,
+      "grad_norm": 2.8395731449127197,
+      "learning_rate": 3.058118081180812e-06,
+      "loss": 1.4647,
+      "step": 1098
+    },
+    {
+      "epoch": 1.3677660236465463,
+      "grad_norm": 2.871156692504883,
+      "learning_rate": 3.0558118081180816e-06,
+      "loss": 1.4196,
+      "step": 1099
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "grad_norm": 2.974905014038086,
+      "learning_rate": 3.053505535055351e-06,
+      "loss": 1.4779,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "eval_loss": 1.651503562927246,
+      "eval_runtime": 54.2713,
+      "eval_samples_per_second": 18.426,
+      "eval_steps_per_second": 0.774,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3702551337896702,
+      "grad_norm": 2.7591328620910645,
+      "learning_rate": 3.05119926199262e-06,
+      "loss": 1.3899,
+      "step": 1101
+    },
+    {
+      "epoch": 1.371499688861232,
+      "grad_norm": 2.8575668334960938,
+      "learning_rate": 3.0488929889298894e-06,
+      "loss": 1.4482,
+      "step": 1102
+    },
+    {
+      "epoch": 1.372744243932794,
+      "grad_norm": 2.9049339294433594,
+      "learning_rate": 3.0465867158671593e-06,
+      "loss": 1.4785,
+      "step": 1103
+    },
+    {
+      "epoch": 1.373988799004356,
+      "grad_norm": 2.816453695297241,
+      "learning_rate": 3.0442804428044283e-06,
+      "loss": 1.4601,
+      "step": 1104
+    },
+    {
+      "epoch": 1.3752333540759178,
+      "grad_norm": 2.8844189643859863,
+      "learning_rate": 3.0419741697416977e-06,
+      "loss": 1.496,
+      "step": 1105
+    },
+    {
+      "epoch": 1.3764779091474797,
+      "grad_norm": 3.014744281768799,
+      "learning_rate": 3.039667896678967e-06,
+      "loss": 1.4098,
+      "step": 1106
+    },
+    {
+      "epoch": 1.3777224642190418,
+      "grad_norm": 2.8243560791015625,
+      "learning_rate": 3.037361623616236e-06,
+      "loss": 1.4086,
+      "step": 1107
+    },
+    {
+      "epoch": 1.3789670192906036,
+      "grad_norm": 2.8333253860473633,
+      "learning_rate": 3.035055350553506e-06,
+      "loss": 1.3516,
+      "step": 1108
+    },
+    {
+      "epoch": 1.3802115743621655,
+      "grad_norm": 2.926455020904541,
+      "learning_rate": 3.0327490774907754e-06,
+      "loss": 1.3976,
+      "step": 1109
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "grad_norm": 2.900937080383301,
+      "learning_rate": 3.0304428044280444e-06,
+      "loss": 1.4336,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "eval_loss": 1.647048830986023,
+      "eval_runtime": 42.3955,
+      "eval_samples_per_second": 23.587,
+      "eval_steps_per_second": 0.991,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3827006845052894,
+      "grad_norm": 3.0756969451904297,
+      "learning_rate": 3.028136531365314e-06,
+      "loss": 1.4674,
+      "step": 1111
+    },
+    {
+      "epoch": 1.3839452395768512,
+      "grad_norm": 2.7754578590393066,
+      "learning_rate": 3.0258302583025832e-06,
+      "loss": 1.4435,
+      "step": 1112
+    },
+    {
+      "epoch": 1.3851897946484133,
+      "grad_norm": 2.8011393547058105,
+      "learning_rate": 3.023523985239853e-06,
+      "loss": 1.4212,
+      "step": 1113
+    },
+    {
+      "epoch": 1.3864343497199751,
+      "grad_norm": 2.7063710689544678,
+      "learning_rate": 3.021217712177122e-06,
+      "loss": 1.3769,
+      "step": 1114
+    },
+    {
+      "epoch": 1.387678904791537,
+      "grad_norm": 2.830411434173584,
+      "learning_rate": 3.0189114391143915e-06,
+      "loss": 1.4832,
+      "step": 1115
+    },
+    {
+      "epoch": 1.3889234598630988,
+      "grad_norm": 2.9075911045074463,
+      "learning_rate": 3.016605166051661e-06,
+      "loss": 1.486,
+      "step": 1116
+    },
+    {
+      "epoch": 1.390168014934661,
+      "grad_norm": 2.847822666168213,
+      "learning_rate": 3.01429889298893e-06,
+      "loss": 1.424,
+      "step": 1117
+    },
+    {
+      "epoch": 1.3914125700062228,
+      "grad_norm": 2.8822033405303955,
+      "learning_rate": 3.0119926199261993e-06,
+      "loss": 1.4218,
+      "step": 1118
+    },
+    {
+      "epoch": 1.3926571250777848,
+      "grad_norm": 2.8944740295410156,
+      "learning_rate": 3.009686346863469e-06,
+      "loss": 1.4283,
+      "step": 1119
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "grad_norm": 2.765190839767456,
+      "learning_rate": 3.007380073800738e-06,
+      "loss": 1.4622,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "eval_loss": 1.6451724767684937,
+      "eval_runtime": 43.7449,
+      "eval_samples_per_second": 22.86,
+      "eval_steps_per_second": 0.96,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3951462352209085,
+      "grad_norm": 2.877002716064453,
+      "learning_rate": 3.0050738007380076e-06,
+      "loss": 1.4145,
+      "step": 1121
+    },
+    {
+      "epoch": 1.3963907902924704,
+      "grad_norm": 2.870389699935913,
+      "learning_rate": 3.002767527675277e-06,
+      "loss": 1.4401,
+      "step": 1122
+    },
+    {
+      "epoch": 1.3976353453640322,
+      "grad_norm": 2.7788920402526855,
+      "learning_rate": 3.000461254612546e-06,
+      "loss": 1.4514,
+      "step": 1123
+    },
+    {
+      "epoch": 1.3988799004355943,
+      "grad_norm": 2.9269332885742188,
+      "learning_rate": 2.998154981549816e-06,
+      "loss": 1.3961,
+      "step": 1124
+    },
+    {
+      "epoch": 1.4001244555071561,
+      "grad_norm": 3.013291835784912,
+      "learning_rate": 2.9958487084870853e-06,
+      "loss": 1.4597,
+      "step": 1125
+    },
+    {
+      "epoch": 1.4013690105787182,
+      "grad_norm": 2.9063880443573,
+      "learning_rate": 2.9935424354243547e-06,
+      "loss": 1.4668,
+      "step": 1126
+    },
+    {
+      "epoch": 1.40261356565028,
+      "grad_norm": 2.8926970958709717,
+      "learning_rate": 2.9912361623616237e-06,
+      "loss": 1.4332,
+      "step": 1127
+    },
+    {
+      "epoch": 1.403858120721842,
+      "grad_norm": 2.8563647270202637,
+      "learning_rate": 2.988929889298893e-06,
+      "loss": 1.407,
+      "step": 1128
+    },
+    {
+      "epoch": 1.4051026757934038,
+      "grad_norm": 2.8725359439849854,
+      "learning_rate": 2.986623616236163e-06,
+      "loss": 1.4434,
+      "step": 1129
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "grad_norm": 2.906898260116577,
+      "learning_rate": 2.984317343173432e-06,
+      "loss": 1.4745,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "eval_loss": 1.647884726524353,
+      "eval_runtime": 44.9221,
+      "eval_samples_per_second": 22.261,
+      "eval_steps_per_second": 0.935,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4075917859365277,
+      "grad_norm": 2.8297719955444336,
+      "learning_rate": 2.9820110701107014e-06,
+      "loss": 1.4542,
+      "step": 1131
+    },
+    {
+      "epoch": 1.4088363410080897,
+      "grad_norm": 2.841031551361084,
+      "learning_rate": 2.979704797047971e-06,
+      "loss": 1.4105,
+      "step": 1132
+    },
+    {
+      "epoch": 1.4100808960796516,
+      "grad_norm": 2.867791175842285,
+      "learning_rate": 2.97739852398524e-06,
+      "loss": 1.4409,
+      "step": 1133
+    },
+    {
+      "epoch": 1.4113254511512134,
+      "grad_norm": 2.8156039714813232,
+      "learning_rate": 2.9750922509225093e-06,
+      "loss": 1.4284,
+      "step": 1134
+    },
+    {
+      "epoch": 1.4125700062227753,
+      "grad_norm": 2.825985908508301,
+      "learning_rate": 2.972785977859779e-06,
+      "loss": 1.4594,
+      "step": 1135
+    },
+    {
+      "epoch": 1.4138145612943374,
+      "grad_norm": 3.0134711265563965,
+      "learning_rate": 2.970479704797048e-06,
+      "loss": 1.4743,
+      "step": 1136
+    },
+    {
+      "epoch": 1.4150591163658992,
+      "grad_norm": 2.9829142093658447,
+      "learning_rate": 2.9681734317343175e-06,
+      "loss": 1.4935,
+      "step": 1137
+    },
+    {
+      "epoch": 1.416303671437461,
+      "grad_norm": 3.067620277404785,
+      "learning_rate": 2.965867158671587e-06,
+      "loss": 1.4266,
+      "step": 1138
+    },
+    {
+      "epoch": 1.4175482265090231,
+      "grad_norm": 2.8969919681549072,
+      "learning_rate": 2.9635608856088564e-06,
+      "loss": 1.451,
+      "step": 1139
+    },
+    {
+      "epoch": 1.418792781580585,
+      "grad_norm": 2.8706412315368652,
+      "learning_rate": 2.9612546125461254e-06,
+      "loss": 1.4834,
+      "step": 1140
+    },
+    {
+      "epoch": 1.418792781580585,
+      "eval_loss": 1.643480658531189,
+      "eval_runtime": 44.743,
+      "eval_samples_per_second": 22.35,
+      "eval_steps_per_second": 0.939,
+      "step": 1140
+    },
+    {
+      "epoch": 1.4200373366521468,
+      "grad_norm": 2.8801848888397217,
+      "learning_rate": 2.958948339483395e-06,
+      "loss": 1.4258,
+      "step": 1141
+    },
+    {
+      "epoch": 1.4212818917237087,
+      "grad_norm": 2.774635076522827,
+      "learning_rate": 2.9566420664206646e-06,
+      "loss": 1.416,
+      "step": 1142
+    },
+    {
+      "epoch": 1.4225264467952707,
+      "grad_norm": 2.8460140228271484,
+      "learning_rate": 2.9543357933579336e-06,
+      "loss": 1.4213,
+      "step": 1143
+    },
+    {
+      "epoch": 1.4237710018668326,
+      "grad_norm": 2.907888889312744,
+      "learning_rate": 2.952029520295203e-06,
+      "loss": 1.4507,
+      "step": 1144
+    },
+    {
+      "epoch": 1.4250155569383947,
+      "grad_norm": 2.789041757583618,
+      "learning_rate": 2.949723247232473e-06,
+      "loss": 1.4257,
+      "step": 1145
+    },
+    {
+      "epoch": 1.4262601120099565,
+      "grad_norm": 2.78859806060791,
+      "learning_rate": 2.947416974169742e-06,
+      "loss": 1.4377,
+      "step": 1146
+    },
+    {
+      "epoch": 1.4275046670815184,
+      "grad_norm": 2.840303421020508,
+      "learning_rate": 2.9451107011070113e-06,
+      "loss": 1.4684,
+      "step": 1147
+    },
+    {
+      "epoch": 1.4287492221530802,
+      "grad_norm": 2.8800530433654785,
+      "learning_rate": 2.9428044280442807e-06,
+      "loss": 1.4912,
+      "step": 1148
+    },
+    {
+      "epoch": 1.4299937772246423,
+      "grad_norm": 2.841092824935913,
+      "learning_rate": 2.9404981549815497e-06,
+      "loss": 1.4234,
+      "step": 1149
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "grad_norm": 2.8226537704467773,
+      "learning_rate": 2.938191881918819e-06,
+      "loss": 1.4578,
+      "step": 1150
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "eval_loss": 1.643249273300171,
+      "eval_runtime": 43.5586,
+      "eval_samples_per_second": 22.958,
+      "eval_steps_per_second": 0.964,
+      "step": 1150
+    },
+    {
+      "epoch": 1.432482887367766,
+      "grad_norm": 2.869736909866333,
+      "learning_rate": 2.935885608856089e-06,
+      "loss": 1.4701,
+      "step": 1151
+    },
+    {
+      "epoch": 1.433727442439328,
+      "grad_norm": 2.689211130142212,
+      "learning_rate": 2.9335793357933584e-06,
+      "loss": 1.4483,
+      "step": 1152
+    },
+    {
+      "epoch": 1.4349719975108899,
+      "grad_norm": 2.9267077445983887,
+      "learning_rate": 2.9312730627306274e-06,
+      "loss": 1.4446,
+      "step": 1153
+    },
+    {
+      "epoch": 1.4362165525824517,
+      "grad_norm": 2.906862497329712,
+      "learning_rate": 2.928966789667897e-06,
+      "loss": 1.4524,
+      "step": 1154
+    },
+    {
+      "epoch": 1.4374611076540136,
+      "grad_norm": 2.8249800205230713,
+      "learning_rate": 2.9266605166051663e-06,
+      "loss": 1.4208,
+      "step": 1155
+    },
+    {
+      "epoch": 1.4387056627255757,
+      "grad_norm": 2.9836032390594482,
+      "learning_rate": 2.9243542435424353e-06,
+      "loss": 1.4581,
+      "step": 1156
+    },
+    {
+      "epoch": 1.4399502177971375,
+      "grad_norm": 2.9595284461975098,
+      "learning_rate": 2.922047970479705e-06,
+      "loss": 1.4141,
+      "step": 1157
+    },
+    {
+      "epoch": 1.4411947728686996,
+      "grad_norm": 3.0363423824310303,
+      "learning_rate": 2.9197416974169746e-06,
+      "loss": 1.4859,
+      "step": 1158
+    },
+    {
+      "epoch": 1.4424393279402614,
+      "grad_norm": 2.911201000213623,
+      "learning_rate": 2.9174354243542436e-06,
+      "loss": 1.4346,
+      "step": 1159
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "grad_norm": 2.8617193698883057,
+      "learning_rate": 2.915129151291513e-06,
+      "loss": 1.433,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "eval_loss": 1.6437232494354248,
+      "eval_runtime": 43.961,
+      "eval_samples_per_second": 22.747,
+      "eval_steps_per_second": 0.955,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4449284380833851,
+      "grad_norm": 2.9584295749664307,
+      "learning_rate": 2.912822878228783e-06,
+      "loss": 1.4847,
+      "step": 1161
+    },
+    {
+      "epoch": 1.4461729931549472,
+      "grad_norm": 2.8209400177001953,
+      "learning_rate": 2.9105166051660522e-06,
+      "loss": 1.421,
+      "step": 1162
+    },
+    {
+      "epoch": 1.447417548226509,
+      "grad_norm": 2.847637414932251,
+      "learning_rate": 2.9082103321033212e-06,
+      "loss": 1.4326,
+      "step": 1163
+    },
+    {
+      "epoch": 1.4486621032980709,
+      "grad_norm": 2.836228132247925,
+      "learning_rate": 2.9059040590405907e-06,
+      "loss": 1.4459,
+      "step": 1164
+    },
+    {
+      "epoch": 1.449906658369633,
+      "grad_norm": 2.8327207565307617,
+      "learning_rate": 2.90359778597786e-06,
+      "loss": 1.4189,
+      "step": 1165
+    },
+    {
+      "epoch": 1.4511512134411948,
+      "grad_norm": 2.880643606185913,
+      "learning_rate": 2.901291512915129e-06,
+      "loss": 1.4214,
+      "step": 1166
+    },
+    {
+      "epoch": 1.4523957685127566,
+      "grad_norm": 2.9637348651885986,
+      "learning_rate": 2.898985239852399e-06,
+      "loss": 1.4471,
+      "step": 1167
+    },
+    {
+      "epoch": 1.4536403235843185,
+      "grad_norm": 3.006145477294922,
+      "learning_rate": 2.8966789667896684e-06,
+      "loss": 1.4228,
+      "step": 1168
+    },
+    {
+      "epoch": 1.4548848786558806,
+      "grad_norm": 2.8547627925872803,
+      "learning_rate": 2.8943726937269374e-06,
+      "loss": 1.4707,
+      "step": 1169
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "grad_norm": 2.973092794418335,
+      "learning_rate": 2.8920664206642068e-06,
+      "loss": 1.4804,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "eval_loss": 1.6340677738189697,
+      "eval_runtime": 44.6094,
+      "eval_samples_per_second": 22.417,
+      "eval_steps_per_second": 0.942,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4573739887990045,
+      "grad_norm": 2.9433603286743164,
+      "learning_rate": 2.889760147601476e-06,
+      "loss": 1.4354,
+      "step": 1171
+    },
+    {
+      "epoch": 1.4586185438705663,
+      "grad_norm": 2.872236728668213,
+      "learning_rate": 2.887453874538745e-06,
+      "loss": 1.418,
+      "step": 1172
+    },
+    {
+      "epoch": 1.4598630989421282,
+      "grad_norm": 2.847557783126831,
+      "learning_rate": 2.885147601476015e-06,
+      "loss": 1.4306,
+      "step": 1173
+    },
+    {
+      "epoch": 1.46110765401369,
+      "grad_norm": 2.7554969787597656,
+      "learning_rate": 2.8828413284132845e-06,
+      "loss": 1.4295,
+      "step": 1174
+    },
+    {
+      "epoch": 1.462352209085252,
+      "grad_norm": 2.8754842281341553,
+      "learning_rate": 2.880535055350554e-06,
+      "loss": 1.4482,
+      "step": 1175
+    },
+    {
+      "epoch": 1.463596764156814,
+      "grad_norm": 3.0115292072296143,
+      "learning_rate": 2.878228782287823e-06,
+      "loss": 1.4428,
+      "step": 1176
+    },
+    {
+      "epoch": 1.4648413192283758,
+      "grad_norm": 2.8976168632507324,
+      "learning_rate": 2.8759225092250923e-06,
+      "loss": 1.4707,
+      "step": 1177
+    },
+    {
+      "epoch": 1.4660858742999379,
+      "grad_norm": 2.7981812953948975,
+      "learning_rate": 2.873616236162362e-06,
+      "loss": 1.4693,
+      "step": 1178
+    },
+    {
+      "epoch": 1.4673304293714997,
+      "grad_norm": 2.8562371730804443,
+      "learning_rate": 2.871309963099631e-06,
+      "loss": 1.4242,
+      "step": 1179
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "grad_norm": 2.8705570697784424,
+      "learning_rate": 2.8690036900369006e-06,
+      "loss": 1.4725,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "eval_loss": 1.6338286399841309,
+      "eval_runtime": 44.7171,
+      "eval_samples_per_second": 22.363,
+      "eval_steps_per_second": 0.939,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4698195395146234,
+      "grad_norm": 2.8025710582733154,
+      "learning_rate": 2.86669741697417e-06,
+      "loss": 1.427,
+      "step": 1181
+    },
+    {
+      "epoch": 1.4710640945861855,
+      "grad_norm": 2.845151901245117,
+      "learning_rate": 2.864391143911439e-06,
+      "loss": 1.4402,
+      "step": 1182
+    },
+    {
+      "epoch": 1.4723086496577473,
+      "grad_norm": 2.943373680114746,
+      "learning_rate": 2.862084870848709e-06,
+      "loss": 1.5194,
+      "step": 1183
+    },
+    {
+      "epoch": 1.4735532047293094,
+      "grad_norm": 2.896955966949463,
+      "learning_rate": 2.8597785977859783e-06,
+      "loss": 1.4242,
+      "step": 1184
+    },
+    {
+      "epoch": 1.4747977598008712,
+      "grad_norm": 3.131305456161499,
+      "learning_rate": 2.8574723247232473e-06,
+      "loss": 1.5008,
+      "step": 1185
+    },
+    {
+      "epoch": 1.476042314872433,
+      "grad_norm": 2.8605735301971436,
+      "learning_rate": 2.8551660516605167e-06,
+      "loss": 1.3533,
+      "step": 1186
+    },
+    {
+      "epoch": 1.477286869943995,
+      "grad_norm": 2.9158811569213867,
+      "learning_rate": 2.852859778597786e-06,
+      "loss": 1.4404,
+      "step": 1187
+    },
+    {
+      "epoch": 1.478531425015557,
+      "grad_norm": 2.7795321941375732,
+      "learning_rate": 2.850553505535056e-06,
+      "loss": 1.3977,
+      "step": 1188
+    },
+    {
+      "epoch": 1.4797759800871189,
+      "grad_norm": 2.8194212913513184,
+      "learning_rate": 2.848247232472325e-06,
+      "loss": 1.4111,
+      "step": 1189
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "grad_norm": 2.983750104904175,
+      "learning_rate": 2.8459409594095944e-06,
+      "loss": 1.4061,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "eval_loss": 1.6375409364700317,
+      "eval_runtime": 45.4026,
+      "eval_samples_per_second": 22.025,
+      "eval_steps_per_second": 0.925,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4822650902302428,
+      "grad_norm": 3.0246665477752686,
+      "learning_rate": 2.843634686346864e-06,
+      "loss": 1.4532,
+      "step": 1191
+    },
+    {
+      "epoch": 1.4835096453018046,
+      "grad_norm": 2.8856449127197266,
+      "learning_rate": 2.841328413284133e-06,
+      "loss": 1.4528,
+      "step": 1192
+    },
+    {
+      "epoch": 1.4847542003733665,
+      "grad_norm": 2.848987102508545,
+      "learning_rate": 2.8390221402214022e-06,
+      "loss": 1.4369,
+      "step": 1193
+    },
+    {
+      "epoch": 1.4859987554449283,
+      "grad_norm": 2.970419406890869,
+      "learning_rate": 2.836715867158672e-06,
+      "loss": 1.4605,
+      "step": 1194
+    },
+    {
+      "epoch": 1.4872433105164904,
+      "grad_norm": 3.156613826751709,
+      "learning_rate": 2.834409594095941e-06,
+      "loss": 1.4489,
+      "step": 1195
+    },
+    {
+      "epoch": 1.4884878655880522,
+      "grad_norm": 3.095349073410034,
+      "learning_rate": 2.8321033210332105e-06,
+      "loss": 1.4604,
+      "step": 1196
+    },
+    {
+      "epoch": 1.4897324206596143,
+      "grad_norm": 2.870742082595825,
+      "learning_rate": 2.82979704797048e-06,
+      "loss": 1.4284,
+      "step": 1197
+    },
+    {
+      "epoch": 1.4909769757311762,
+      "grad_norm": 2.9415056705474854,
+      "learning_rate": 2.827490774907749e-06,
+      "loss": 1.4715,
+      "step": 1198
+    },
+    {
+      "epoch": 1.492221530802738,
+      "grad_norm": 2.8297767639160156,
+      "learning_rate": 2.8251845018450188e-06,
+      "loss": 1.4508,
+      "step": 1199
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "grad_norm": 2.872086524963379,
+      "learning_rate": 2.822878228782288e-06,
+      "loss": 1.5098,
+      "step": 1200
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "eval_loss": 1.6357526779174805,
+      "eval_runtime": 51.422,
+      "eval_samples_per_second": 19.447,
+      "eval_steps_per_second": 0.817,
+      "step": 1200
+    },
+    {
+      "epoch": 1.494710640945862,
+      "grad_norm": 2.7152082920074463,
+      "learning_rate": 2.8205719557195576e-06,
+      "loss": 1.4329,
+      "step": 1201
+    },
+    {
+      "epoch": 1.4959551960174238,
+      "grad_norm": 2.866170883178711,
+      "learning_rate": 2.8182656826568266e-06,
+      "loss": 1.4455,
+      "step": 1202
+    },
+    {
+      "epoch": 1.4971997510889856,
+      "grad_norm": 2.9010684490203857,
+      "learning_rate": 2.815959409594096e-06,
+      "loss": 1.423,
+      "step": 1203
+    },
+    {
+      "epoch": 1.4984443061605477,
+      "grad_norm": 2.8588502407073975,
+      "learning_rate": 2.813653136531366e-06,
+      "loss": 1.4041,
+      "step": 1204
+    },
+    {
+      "epoch": 1.4996888612321095,
+      "grad_norm": 2.932544231414795,
+      "learning_rate": 2.811346863468635e-06,
+      "loss": 1.4559,
+      "step": 1205
+    },
+    {
+      "epoch": 1.5009334163036714,
+      "grad_norm": 2.7486014366149902,
+      "learning_rate": 2.8090405904059043e-06,
+      "loss": 1.4337,
+      "step": 1206
+    },
+    {
+      "epoch": 1.5021779713752332,
+      "grad_norm": 3.0866589546203613,
+      "learning_rate": 2.8067343173431737e-06,
+      "loss": 1.4186,
+      "step": 1207
+    },
+    {
+      "epoch": 1.5034225264467953,
+      "grad_norm": 2.9710276126861572,
+      "learning_rate": 2.8044280442804427e-06,
+      "loss": 1.419,
+      "step": 1208
+    },
+    {
+      "epoch": 1.5046670815183572,
+      "grad_norm": 2.8379950523376465,
+      "learning_rate": 2.802121771217712e-06,
+      "loss": 1.4922,
+      "step": 1209
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "grad_norm": 2.939629554748535,
+      "learning_rate": 2.799815498154982e-06,
+      "loss": 1.4402,
+      "step": 1210
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "eval_loss": 1.6419442892074585,
+      "eval_runtime": 43.1202,
+      "eval_samples_per_second": 23.191,
+      "eval_steps_per_second": 0.974,
+      "step": 1210
+    },
+    {
+      "epoch": 1.507156191661481,
+      "grad_norm": 3.041389226913452,
+      "learning_rate": 2.7975092250922514e-06,
+      "loss": 1.4873,
+      "step": 1211
+    },
+    {
+      "epoch": 1.508400746733043,
+      "grad_norm": 2.9776995182037354,
+      "learning_rate": 2.7952029520295204e-06,
+      "loss": 1.4422,
+      "step": 1212
+    },
+    {
+      "epoch": 1.5096453018046048,
+      "grad_norm": 2.8798792362213135,
+      "learning_rate": 2.79289667896679e-06,
+      "loss": 1.4535,
+      "step": 1213
+    },
+    {
+      "epoch": 1.5108898568761666,
+      "grad_norm": 2.9016385078430176,
+      "learning_rate": 2.7905904059040597e-06,
+      "loss": 1.4462,
+      "step": 1214
+    },
+    {
+      "epoch": 1.5121344119477287,
+      "grad_norm": 2.8987581729888916,
+      "learning_rate": 2.7882841328413287e-06,
+      "loss": 1.4477,
+      "step": 1215
+    },
+    {
+      "epoch": 1.5133789670192908,
+      "grad_norm": 2.960266351699829,
+      "learning_rate": 2.785977859778598e-06,
+      "loss": 1.4811,
+      "step": 1216
+    },
+    {
+      "epoch": 1.5146235220908526,
+      "grad_norm": 2.8965611457824707,
+      "learning_rate": 2.7836715867158675e-06,
+      "loss": 1.4366,
+      "step": 1217
+    },
+    {
+      "epoch": 1.5158680771624145,
+      "grad_norm": 3.0667364597320557,
+      "learning_rate": 2.7813653136531365e-06,
+      "loss": 1.3828,
+      "step": 1218
+    },
+    {
+      "epoch": 1.5171126322339763,
+      "grad_norm": 2.952362537384033,
+      "learning_rate": 2.779059040590406e-06,
+      "loss": 1.3903,
+      "step": 1219
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "grad_norm": 2.9899795055389404,
+      "learning_rate": 2.776752767527676e-06,
+      "loss": 1.4312,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "eval_loss": 1.640383005142212,
+      "eval_runtime": 45.4128,
+      "eval_samples_per_second": 22.02,
+      "eval_steps_per_second": 0.925,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5196017423771002,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4344,
+      "step": 1221
+    },
+    {
+      "epoch": 1.520846297448662,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4386,
+      "step": 1222
+    },
+    {
+      "epoch": 1.5220908525202241,
+      "grad_norm": 2.883223295211792,
+      "learning_rate": 2.7721402214022142e-06,
+      "loss": 1.4401,
+      "step": 1223
+    },
+    {
+      "epoch": 1.523335407591786,
+      "grad_norm": 2.8346424102783203,
+      "learning_rate": 2.7698339483394837e-06,
+      "loss": 1.3926,
+      "step": 1224
+    },
+    {
+      "epoch": 1.5245799626633478,
+      "grad_norm": 2.8898870944976807,
+      "learning_rate": 2.767527675276753e-06,
+      "loss": 1.4128,
+      "step": 1225
+    },
+    {
+      "epoch": 1.5258245177349097,
+      "grad_norm": 2.828594923019409,
+      "learning_rate": 2.765221402214022e-06,
+      "loss": 1.4103,
+      "step": 1226
+    },
+    {
+      "epoch": 1.5270690728064715,
+      "grad_norm": 2.882558822631836,
+      "learning_rate": 2.762915129151292e-06,
+      "loss": 1.4186,
+      "step": 1227
+    },
+    {
+      "epoch": 1.5283136278780336,
+      "grad_norm": 2.955760955810547,
+      "learning_rate": 2.7606088560885613e-06,
+      "loss": 1.3705,
+      "step": 1228
+    },
+    {
+      "epoch": 1.5295581829495957,
+      "grad_norm": 2.862934112548828,
+      "learning_rate": 2.7583025830258303e-06,
+      "loss": 1.5012,
+      "step": 1229
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "grad_norm": 2.926501989364624,
+      "learning_rate": 2.7559963099630998e-06,
+      "loss": 1.3875,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "eval_loss": 1.6376254558563232,
+      "eval_runtime": 47.9535,
+      "eval_samples_per_second": 20.854,
+      "eval_steps_per_second": 0.876,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5320472930927194,
+      "grad_norm": 2.7648823261260986,
+      "learning_rate": 2.753690036900369e-06,
+      "loss": 1.3781,
+      "step": 1231
+    },
+    {
+      "epoch": 1.5332918481642812,
+      "grad_norm": 2.927103281021118,
+      "learning_rate": 2.751383763837638e-06,
+      "loss": 1.4091,
+      "step": 1232
+    },
+    {
+      "epoch": 1.534536403235843,
+      "grad_norm": 2.964823007583618,
+      "learning_rate": 2.749077490774908e-06,
+      "loss": 1.5075,
+      "step": 1233
+    },
+    {
+      "epoch": 1.5357809583074051,
+      "grad_norm": 2.7944016456604004,
+      "learning_rate": 2.7467712177121775e-06,
+      "loss": 1.3888,
+      "step": 1234
+    },
+    {
+      "epoch": 1.537025513378967,
+      "grad_norm": 2.891592025756836,
+      "learning_rate": 2.7444649446494465e-06,
+      "loss": 1.4825,
+      "step": 1235
+    },
+    {
+      "epoch": 1.538270068450529,
+      "grad_norm": 2.8120031356811523,
+      "learning_rate": 2.742158671586716e-06,
+      "loss": 1.4246,
+      "step": 1236
+    },
+    {
+      "epoch": 1.539514623522091,
+      "grad_norm": 2.9384255409240723,
+      "learning_rate": 2.7398523985239857e-06,
+      "loss": 1.4657,
+      "step": 1237
+    },
+    {
+      "epoch": 1.5407591785936527,
+      "grad_norm": 2.73111891746521,
+      "learning_rate": 2.737546125461255e-06,
+      "loss": 1.4689,
+      "step": 1238
+    },
+    {
+      "epoch": 1.5420037336652146,
+      "grad_norm": 2.858963966369629,
+      "learning_rate": 2.735239852398524e-06,
+      "loss": 1.4626,
+      "step": 1239
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "grad_norm": 2.9148507118225098,
+      "learning_rate": 2.7329335793357936e-06,
+      "loss": 1.4787,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "eval_loss": 1.6436412334442139,
+      "eval_runtime": 47.3139,
+      "eval_samples_per_second": 21.135,
+      "eval_steps_per_second": 0.888,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5444928438083385,
+      "grad_norm": 2.8731327056884766,
+      "learning_rate": 2.730627306273063e-06,
+      "loss": 1.4748,
+      "step": 1241
+    },
+    {
+      "epoch": 1.5457373988799006,
+      "grad_norm": 2.990265369415283,
+      "learning_rate": 2.728321033210332e-06,
+      "loss": 1.4375,
+      "step": 1242
+    },
+    {
+      "epoch": 1.5469819539514624,
+      "grad_norm": 2.874321460723877,
+      "learning_rate": 2.726014760147602e-06,
+      "loss": 1.4154,
+      "step": 1243
+    },
+    {
+      "epoch": 1.5482265090230243,
+      "grad_norm": 2.9316225051879883,
+      "learning_rate": 2.7237084870848713e-06,
+      "loss": 1.4285,
+      "step": 1244
+    },
+    {
+      "epoch": 1.5494710640945861,
+      "grad_norm": 2.783966302871704,
+      "learning_rate": 2.7214022140221403e-06,
+      "loss": 1.409,
+      "step": 1245
+    },
+    {
+      "epoch": 1.550715619166148,
+      "grad_norm": 2.9578707218170166,
+      "learning_rate": 2.7190959409594097e-06,
+      "loss": 1.4325,
+      "step": 1246
+    },
+    {
+      "epoch": 1.55196017423771,
+      "grad_norm": 2.869893789291382,
+      "learning_rate": 2.716789667896679e-06,
+      "loss": 1.3989,
+      "step": 1247
+    },
+    {
+      "epoch": 1.553204729309272,
+      "grad_norm": 2.962944269180298,
+      "learning_rate": 2.714483394833948e-06,
+      "loss": 1.4302,
+      "step": 1248
+    },
+    {
+      "epoch": 1.554449284380834,
+      "grad_norm": 2.974494695663452,
+      "learning_rate": 2.712177121771218e-06,
+      "loss": 1.46,
+      "step": 1249
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "grad_norm": 2.9154603481292725,
+      "learning_rate": 2.7098708487084874e-06,
+      "loss": 1.4388,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "eval_loss": 1.6357773542404175,
+      "eval_runtime": 47.6085,
+      "eval_samples_per_second": 21.005,
+      "eval_steps_per_second": 0.882,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5569383945239577,
+      "grad_norm": 3.0333547592163086,
+      "learning_rate": 2.707564575645757e-06,
+      "loss": 1.4846,
+      "step": 1251
+    },
+    {
+      "epoch": 1.5581829495955195,
+      "grad_norm": 3.06097674369812,
+      "learning_rate": 2.705258302583026e-06,
+      "loss": 1.4355,
+      "step": 1252
+    },
+    {
+      "epoch": 1.5594275046670814,
+      "grad_norm": 2.872680425643921,
+      "learning_rate": 2.7029520295202956e-06,
+      "loss": 1.4219,
+      "step": 1253
+    },
+    {
+      "epoch": 1.5606720597386434,
+      "grad_norm": 2.9066731929779053,
+      "learning_rate": 2.700645756457565e-06,
+      "loss": 1.384,
+      "step": 1254
+    },
+    {
+      "epoch": 1.5619166148102055,
+      "grad_norm": 3.0570194721221924,
+      "learning_rate": 2.698339483394834e-06,
+      "loss": 1.4687,
+      "step": 1255
+    },
+    {
+      "epoch": 1.5631611698817673,
+      "grad_norm": 3.1955857276916504,
+      "learning_rate": 2.6960332103321035e-06,
+      "loss": 1.4736,
+      "step": 1256
+    },
+    {
+      "epoch": 1.5644057249533292,
+      "grad_norm": 2.9686198234558105,
+      "learning_rate": 2.693726937269373e-06,
+      "loss": 1.4176,
+      "step": 1257
+    },
+    {
+      "epoch": 1.565650280024891,
+      "grad_norm": 3.1161272525787354,
+      "learning_rate": 2.691420664206642e-06,
+      "loss": 1.443,
+      "step": 1258
+    },
+    {
+      "epoch": 1.5668948350964529,
+      "grad_norm": 2.9031193256378174,
+      "learning_rate": 2.6891143911439118e-06,
+      "loss": 1.3871,
+      "step": 1259
+    },
+    {
+      "epoch": 1.568139390168015,
+      "grad_norm": 2.9120476245880127,
+      "learning_rate": 2.686808118081181e-06,
+      "loss": 1.4406,
+      "step": 1260
+    },
+    {
+      "epoch": 1.568139390168015,
+      "eval_loss": 1.6291446685791016,
+      "eval_runtime": 49.7006,
+      "eval_samples_per_second": 20.12,
+      "eval_steps_per_second": 0.845,
+      "step": 1260
+    },
+    {
+      "epoch": 1.5693839452395768,
+      "grad_norm": 3.3181824684143066,
+      "learning_rate": 2.6845018450184506e-06,
+      "loss": 1.5278,
+      "step": 1261
+    },
+    {
+      "epoch": 1.5706285003111389,
+      "grad_norm": 2.9628148078918457,
+      "learning_rate": 2.6821955719557196e-06,
+      "loss": 1.4206,
+      "step": 1262
+    },
+    {
+      "epoch": 1.5718730553827007,
+      "grad_norm": 2.9828474521636963,
+      "learning_rate": 2.679889298892989e-06,
+      "loss": 1.5478,
+      "step": 1263
+    },
+    {
+      "epoch": 1.5731176104542626,
+      "grad_norm": 3.018711805343628,
+      "learning_rate": 2.677583025830259e-06,
+      "loss": 1.4442,
+      "step": 1264
+    },
+    {
+      "epoch": 1.5743621655258244,
+      "grad_norm": 2.8596792221069336,
+      "learning_rate": 2.675276752767528e-06,
+      "loss": 1.4325,
+      "step": 1265
+    },
+    {
+      "epoch": 1.5756067205973865,
+      "grad_norm": 2.9592769145965576,
+      "learning_rate": 2.6729704797047973e-06,
+      "loss": 1.4059,
+      "step": 1266
+    },
+    {
+      "epoch": 1.5768512756689483,
+      "grad_norm": 2.773589611053467,
+      "learning_rate": 2.6706642066420667e-06,
+      "loss": 1.3608,
+      "step": 1267
+    },
+    {
+      "epoch": 1.5780958307405104,
+      "grad_norm": 3.313255548477173,
+      "learning_rate": 2.6683579335793357e-06,
+      "loss": 1.4552,
+      "step": 1268
+    },
+    {
+      "epoch": 1.5793403858120723,
+      "grad_norm": 2.977649211883545,
+      "learning_rate": 2.6660516605166056e-06,
+      "loss": 1.4221,
+      "step": 1269
+    },
+    {
+      "epoch": 1.580584940883634,
+      "grad_norm": 3.0672051906585693,
+      "learning_rate": 2.663745387453875e-06,
+      "loss": 1.4568,
+      "step": 1270
+    },
+    {
+      "epoch": 1.580584940883634,
+      "eval_loss": 1.6317907571792603,
+      "eval_runtime": 47.2354,
+      "eval_samples_per_second": 21.171,
+      "eval_steps_per_second": 0.889,
+      "step": 1270
+    },
+    {
+      "epoch": 1.581829495955196,
+      "grad_norm": 3.029320240020752,
+      "learning_rate": 2.661439114391144e-06,
+      "loss": 1.4432,
+      "step": 1271
+    },
+    {
+      "epoch": 1.5830740510267578,
+      "grad_norm": 3.0183846950531006,
+      "learning_rate": 2.6591328413284134e-06,
+      "loss": 1.4332,
+      "step": 1272
+    },
+    {
+      "epoch": 1.5843186060983199,
+      "grad_norm": 2.856551170349121,
+      "learning_rate": 2.656826568265683e-06,
+      "loss": 1.4291,
+      "step": 1273
+    },
+    {
+      "epoch": 1.5855631611698817,
+      "grad_norm": 3.124971389770508,
+      "learning_rate": 2.6545202952029527e-06,
+      "loss": 1.4363,
+      "step": 1274
+    },
+    {
+      "epoch": 1.5868077162414438,
+      "grad_norm": 2.9470791816711426,
+      "learning_rate": 2.6522140221402217e-06,
+      "loss": 1.4502,
+      "step": 1275
+    },
+    {
+      "epoch": 1.5880522713130056,
+      "grad_norm": 2.9116358757019043,
+      "learning_rate": 2.649907749077491e-06,
+      "loss": 1.4314,
+      "step": 1276
+    },
+    {
+      "epoch": 1.5892968263845675,
+      "grad_norm": 3.2575323581695557,
+      "learning_rate": 2.6476014760147605e-06,
+      "loss": 1.4749,
+      "step": 1277
+    },
+    {
+      "epoch": 1.5905413814561293,
+      "grad_norm": 2.935518503189087,
+      "learning_rate": 2.6452952029520295e-06,
+      "loss": 1.4265,
+      "step": 1278
+    },
+    {
+      "epoch": 1.5917859365276914,
+      "grad_norm": 2.9750072956085205,
+      "learning_rate": 2.642988929889299e-06,
+      "loss": 1.4501,
+      "step": 1279
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "grad_norm": 2.8086845874786377,
+      "learning_rate": 2.640682656826569e-06,
+      "loss": 1.4557,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "eval_loss": 1.6302106380462646,
+      "eval_runtime": 52.3962,
+      "eval_samples_per_second": 19.085,
+      "eval_steps_per_second": 0.802,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5942750466708153,
+      "grad_norm": 2.741849422454834,
+      "learning_rate": 2.638376383763838e-06,
+      "loss": 1.3918,
+      "step": 1281
+    },
+    {
+      "epoch": 1.5955196017423772,
+      "grad_norm": 2.9414806365966797,
+      "learning_rate": 2.6360701107011072e-06,
+      "loss": 1.3882,
+      "step": 1282
+    },
+    {
+      "epoch": 1.596764156813939,
+      "grad_norm": 2.901514768600464,
+      "learning_rate": 2.6337638376383766e-06,
+      "loss": 1.4331,
+      "step": 1283
+    },
+    {
+      "epoch": 1.5980087118855009,
+      "grad_norm": 2.8159122467041016,
+      "learning_rate": 2.6314575645756456e-06,
+      "loss": 1.4064,
+      "step": 1284
+    },
+    {
+      "epoch": 1.5992532669570627,
+      "grad_norm": 2.900561571121216,
+      "learning_rate": 2.629151291512915e-06,
+      "loss": 1.3731,
+      "step": 1285
+    },
+    {
+      "epoch": 1.6004978220286248,
+      "grad_norm": 3.019540548324585,
+      "learning_rate": 2.626845018450185e-06,
+      "loss": 1.4506,
+      "step": 1286
+    },
+    {
+      "epoch": 1.6017423771001866,
+      "grad_norm": 2.9263620376586914,
+      "learning_rate": 2.6245387453874543e-06,
+      "loss": 1.4754,
+      "step": 1287
+    },
+    {
+      "epoch": 1.6029869321717487,
+      "grad_norm": 2.8681881427764893,
+      "learning_rate": 2.6222324723247233e-06,
+      "loss": 1.4124,
+      "step": 1288
+    },
+    {
+      "epoch": 1.6042314872433105,
+      "grad_norm": 2.922945261001587,
+      "learning_rate": 2.6199261992619928e-06,
+      "loss": 1.5101,
+      "step": 1289
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "grad_norm": 2.9548606872558594,
+      "learning_rate": 2.6176199261992626e-06,
+      "loss": 1.41,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "eval_loss": 1.6357502937316895,
+      "eval_runtime": 46.9547,
+      "eval_samples_per_second": 21.297,
+      "eval_steps_per_second": 0.894,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6067205973864342,
+      "grad_norm": 2.837184190750122,
+      "learning_rate": 2.6153136531365316e-06,
+      "loss": 1.4354,
+      "step": 1291
+    },
+    {
+      "epoch": 1.6079651524579963,
+      "grad_norm": 2.818990468978882,
+      "learning_rate": 2.613007380073801e-06,
+      "loss": 1.4474,
+      "step": 1292
+    },
+    {
+      "epoch": 1.6092097075295582,
+      "grad_norm": 2.7880146503448486,
+      "learning_rate": 2.6107011070110704e-06,
+      "loss": 1.4156,
+      "step": 1293
+    },
+    {
+      "epoch": 1.6104542626011202,
+      "grad_norm": 2.790971040725708,
+      "learning_rate": 2.6083948339483394e-06,
+      "loss": 1.4671,
+      "step": 1294
+    },
+    {
+      "epoch": 1.611698817672682,
+      "grad_norm": 2.943129777908325,
+      "learning_rate": 2.606088560885609e-06,
+      "loss": 1.4801,
+      "step": 1295
+    },
+    {
+      "epoch": 1.612943372744244,
+      "grad_norm": 2.8589110374450684,
+      "learning_rate": 2.6037822878228787e-06,
+      "loss": 1.4894,
+      "step": 1296
+    },
+    {
+      "epoch": 1.6141879278158058,
+      "grad_norm": 2.8929262161254883,
+      "learning_rate": 2.6014760147601477e-06,
+      "loss": 1.4022,
+      "step": 1297
+    },
+    {
+      "epoch": 1.6154324828873676,
+      "grad_norm": 2.970099925994873,
+      "learning_rate": 2.599169741697417e-06,
+      "loss": 1.5136,
+      "step": 1298
+    },
+    {
+      "epoch": 1.6166770379589297,
+      "grad_norm": 2.928466796875,
+      "learning_rate": 2.5968634686346866e-06,
+      "loss": 1.4743,
+      "step": 1299
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "grad_norm": 2.941580057144165,
+      "learning_rate": 2.594557195571956e-06,
+      "loss": 1.4397,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "eval_loss": 1.6348339319229126,
+      "eval_runtime": 50.433,
+      "eval_samples_per_second": 19.828,
+      "eval_steps_per_second": 0.833,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6191661481020536,
+      "grad_norm": 2.903212308883667,
+      "learning_rate": 2.592250922509225e-06,
+      "loss": 1.4221,
+      "step": 1301
+    },
+    {
+      "epoch": 1.6204107031736155,
+      "grad_norm": 2.948651075363159,
+      "learning_rate": 2.589944649446495e-06,
+      "loss": 1.4363,
+      "step": 1302
+    },
+    {
+      "epoch": 1.6216552582451773,
+      "grad_norm": 2.8940505981445312,
+      "learning_rate": 2.5876383763837643e-06,
+      "loss": 1.398,
+      "step": 1303
+    },
+    {
+      "epoch": 1.6228998133167392,
+      "grad_norm": 2.950056552886963,
+      "learning_rate": 2.5853321033210333e-06,
+      "loss": 1.4425,
+      "step": 1304
+    },
+    {
+      "epoch": 1.6241443683883012,
+      "grad_norm": 2.9674088954925537,
+      "learning_rate": 2.5830258302583027e-06,
+      "loss": 1.4473,
+      "step": 1305
+    },
+    {
+      "epoch": 1.625388923459863,
+      "grad_norm": 2.7541685104370117,
+      "learning_rate": 2.5807195571955725e-06,
+      "loss": 1.4483,
+      "step": 1306
+    },
+    {
+      "epoch": 1.6266334785314251,
+      "grad_norm": 2.8897035121917725,
+      "learning_rate": 2.5784132841328415e-06,
+      "loss": 1.4572,
+      "step": 1307
+    },
+    {
+      "epoch": 1.627878033602987,
+      "grad_norm": 3.021559238433838,
+      "learning_rate": 2.576107011070111e-06,
+      "loss": 1.5164,
+      "step": 1308
+    },
+    {
+      "epoch": 1.6291225886745488,
+      "grad_norm": 2.884009838104248,
+      "learning_rate": 2.5738007380073804e-06,
+      "loss": 1.4196,
+      "step": 1309
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "grad_norm": 2.749497890472412,
+      "learning_rate": 2.5714944649446494e-06,
+      "loss": 1.4324,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "eval_loss": 1.6314265727996826,
+      "eval_runtime": 42.4647,
+      "eval_samples_per_second": 23.549,
+      "eval_steps_per_second": 0.989,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6316116988176725,
+      "grad_norm": 2.8908045291900635,
+      "learning_rate": 2.5691881918819188e-06,
+      "loss": 1.4388,
+      "step": 1311
+    },
+    {
+      "epoch": 1.6328562538892346,
+      "grad_norm": 2.832808256149292,
+      "learning_rate": 2.5668819188191886e-06,
+      "loss": 1.4368,
+      "step": 1312
+    },
+    {
+      "epoch": 1.6341008089607967,
+      "grad_norm": 2.9047906398773193,
+      "learning_rate": 2.564575645756458e-06,
+      "loss": 1.4186,
+      "step": 1313
+    },
+    {
+      "epoch": 1.6353453640323585,
+      "grad_norm": 2.9271419048309326,
+      "learning_rate": 2.562269372693727e-06,
+      "loss": 1.3776,
+      "step": 1314
+    },
+    {
+      "epoch": 1.6365899191039204,
+      "grad_norm": 2.760314702987671,
+      "learning_rate": 2.5599630996309965e-06,
+      "loss": 1.386,
+      "step": 1315
+    },
+    {
+      "epoch": 1.6378344741754822,
+      "grad_norm": 2.9694864749908447,
+      "learning_rate": 2.557656826568266e-06,
+      "loss": 1.4542,
+      "step": 1316
+    },
+    {
+      "epoch": 1.639079029247044,
+      "grad_norm": 2.9950435161590576,
+      "learning_rate": 2.555350553505535e-06,
+      "loss": 1.4242,
+      "step": 1317
+    },
+    {
+      "epoch": 1.6403235843186061,
+      "grad_norm": 2.9291114807128906,
+      "learning_rate": 2.5530442804428047e-06,
+      "loss": 1.4289,
+      "step": 1318
+    },
+    {
+      "epoch": 1.641568139390168,
+      "grad_norm": 2.981961488723755,
+      "learning_rate": 2.550738007380074e-06,
+      "loss": 1.4283,
+      "step": 1319
+    },
+    {
+      "epoch": 1.64281269446173,
+      "grad_norm": 2.734123706817627,
+      "learning_rate": 2.548431734317343e-06,
+      "loss": 1.4144,
+      "step": 1320
+    },
+    {
+      "epoch": 1.64281269446173,
+      "eval_loss": 1.6249239444732666,
+      "eval_runtime": 48.8116,
+      "eval_samples_per_second": 20.487,
+      "eval_steps_per_second": 0.86,
+      "step": 1320
+    },
+    {
+      "epoch": 1.644057249533292,
+      "grad_norm": 2.9752919673919678,
+      "learning_rate": 2.5461254612546126e-06,
+      "loss": 1.3641,
+      "step": 1321
+    },
+    {
+      "epoch": 1.6453018046048538,
+      "grad_norm": 2.972463607788086,
+      "learning_rate": 2.543819188191882e-06,
+      "loss": 1.3797,
+      "step": 1322
+    },
+    {
+      "epoch": 1.6465463596764156,
+      "grad_norm": 2.905499219894409,
+      "learning_rate": 2.541512915129152e-06,
+      "loss": 1.4221,
+      "step": 1323
+    },
+    {
+      "epoch": 1.6477909147479775,
+      "grad_norm": 2.7921793460845947,
+      "learning_rate": 2.539206642066421e-06,
+      "loss": 1.4118,
+      "step": 1324
+    },
+    {
+      "epoch": 1.6490354698195395,
+      "grad_norm": 3.064558267593384,
+      "learning_rate": 2.5369003690036903e-06,
+      "loss": 1.4611,
+      "step": 1325
+    },
+    {
+      "epoch": 1.6502800248911016,
+      "grad_norm": 2.9690096378326416,
+      "learning_rate": 2.5345940959409597e-06,
+      "loss": 1.461,
+      "step": 1326
+    },
+    {
+      "epoch": 1.6515245799626634,
+      "grad_norm": 3.0379040241241455,
+      "learning_rate": 2.5322878228782287e-06,
+      "loss": 1.451,
+      "step": 1327
+    },
+    {
+      "epoch": 1.6527691350342253,
+      "grad_norm": 2.9426493644714355,
+      "learning_rate": 2.5299815498154986e-06,
+      "loss": 1.4671,
+      "step": 1328
+    },
+    {
+      "epoch": 1.6540136901057871,
+      "grad_norm": 2.915076732635498,
+      "learning_rate": 2.527675276752768e-06,
+      "loss": 1.4159,
+      "step": 1329
+    },
+    {
+      "epoch": 1.655258245177349,
+      "grad_norm": 2.8588879108428955,
+      "learning_rate": 2.525369003690037e-06,
+      "loss": 1.4978,
+      "step": 1330
+    },
+    {
+      "epoch": 1.655258245177349,
+      "eval_loss": 1.624009132385254,
+      "eval_runtime": 47.0461,
+      "eval_samples_per_second": 21.256,
+      "eval_steps_per_second": 0.893,
+      "step": 1330
+    },
+    {
+      "epoch": 1.656502800248911,
+      "grad_norm": 2.9942209720611572,
+      "learning_rate": 2.5230627306273064e-06,
+      "loss": 1.3872,
+      "step": 1331
+    },
+    {
+      "epoch": 1.657747355320473,
+      "grad_norm": 2.8424410820007324,
+      "learning_rate": 2.520756457564576e-06,
+      "loss": 1.4181,
+      "step": 1332
+    },
+    {
+      "epoch": 1.658991910392035,
+      "grad_norm": 2.9234843254089355,
+      "learning_rate": 2.518450184501845e-06,
+      "loss": 1.4516,
+      "step": 1333
+    },
+    {
+      "epoch": 1.6602364654635968,
+      "grad_norm": 2.8367815017700195,
+      "learning_rate": 2.5161439114391147e-06,
+      "loss": 1.4214,
+      "step": 1334
+    },
+    {
+      "epoch": 1.6614810205351587,
+      "grad_norm": 2.819149971008301,
+      "learning_rate": 2.513837638376384e-06,
+      "loss": 1.3883,
+      "step": 1335
+    },
+    {
+      "epoch": 1.6627255756067205,
+      "grad_norm": 3.024693250656128,
+      "learning_rate": 2.5115313653136535e-06,
+      "loss": 1.3902,
+      "step": 1336
+    },
+    {
+      "epoch": 1.6639701306782824,
+      "grad_norm": 2.903599262237549,
+      "learning_rate": 2.5092250922509225e-06,
+      "loss": 1.4238,
+      "step": 1337
+    },
+    {
+      "epoch": 1.6652146857498444,
+      "grad_norm": 2.8296589851379395,
+      "learning_rate": 2.506918819188192e-06,
+      "loss": 1.3993,
+      "step": 1338
+    },
+    {
+      "epoch": 1.6664592408214065,
+      "grad_norm": 2.873272657394409,
+      "learning_rate": 2.5046125461254618e-06,
+      "loss": 1.4186,
+      "step": 1339
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "grad_norm": 2.9268789291381836,
+      "learning_rate": 2.5023062730627308e-06,
+      "loss": 1.4242,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "eval_loss": 1.6260672807693481,
+      "eval_runtime": 47.5895,
+      "eval_samples_per_second": 21.013,
+      "eval_steps_per_second": 0.883,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6689483509645302,
+      "grad_norm": 3.057659387588501,
+      "learning_rate": 2.5e-06,
+      "loss": 1.4077,
+      "step": 1341
+    },
+    {
+      "epoch": 1.670192906036092,
+      "grad_norm": 2.875095844268799,
+      "learning_rate": 2.4976937269372696e-06,
+      "loss": 1.394,
+      "step": 1342
+    },
+    {
+      "epoch": 1.671437461107654,
+      "grad_norm": 2.7579872608184814,
+      "learning_rate": 2.495387453874539e-06,
+      "loss": 1.4029,
+      "step": 1343
+    },
+    {
+      "epoch": 1.672682016179216,
+      "grad_norm": 2.8981001377105713,
+      "learning_rate": 2.4930811808118085e-06,
+      "loss": 1.4494,
+      "step": 1344
+    },
+    {
+      "epoch": 1.6739265712507778,
+      "grad_norm": 2.9696900844573975,
+      "learning_rate": 2.490774907749078e-06,
+      "loss": 1.4692,
+      "step": 1345
+    },
+    {
+      "epoch": 1.6751711263223399,
+      "grad_norm": 2.8094892501831055,
+      "learning_rate": 2.488468634686347e-06,
+      "loss": 1.441,
+      "step": 1346
+    },
+    {
+      "epoch": 1.6764156813939017,
+      "grad_norm": 2.8619563579559326,
+      "learning_rate": 2.4861623616236163e-06,
+      "loss": 1.4885,
+      "step": 1347
+    },
+    {
+      "epoch": 1.6776602364654636,
+      "grad_norm": 2.854151487350464,
+      "learning_rate": 2.4838560885608857e-06,
+      "loss": 1.4338,
+      "step": 1348
+    },
+    {
+      "epoch": 1.6789047915370254,
+      "grad_norm": 2.9707868099212646,
+      "learning_rate": 2.481549815498155e-06,
+      "loss": 1.4529,
+      "step": 1349
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "grad_norm": 3.0289969444274902,
+      "learning_rate": 2.4792435424354246e-06,
+      "loss": 1.4855,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "eval_loss": 1.6256393194198608,
+      "eval_runtime": 45.9524,
+      "eval_samples_per_second": 21.762,
+      "eval_steps_per_second": 0.914,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6813939016801493,
+      "grad_norm": 2.8407845497131348,
+      "learning_rate": 2.476937269372694e-06,
+      "loss": 1.3876,
+      "step": 1351
+    },
+    {
+      "epoch": 1.6826384567517114,
+      "grad_norm": 2.9876599311828613,
+      "learning_rate": 2.4746309963099634e-06,
+      "loss": 1.3931,
+      "step": 1352
+    },
+    {
+      "epoch": 1.6838830118232733,
+      "grad_norm": 3.008683919906616,
+      "learning_rate": 2.472324723247233e-06,
+      "loss": 1.4482,
+      "step": 1353
+    },
+    {
+      "epoch": 1.6851275668948351,
+      "grad_norm": 2.8110079765319824,
+      "learning_rate": 2.470018450184502e-06,
+      "loss": 1.3851,
+      "step": 1354
+    },
+    {
+      "epoch": 1.686372121966397,
+      "grad_norm": 2.9402263164520264,
+      "learning_rate": 2.4677121771217713e-06,
+      "loss": 1.4009,
+      "step": 1355
+    },
+    {
+      "epoch": 1.6876166770379588,
+      "grad_norm": 2.9846489429473877,
+      "learning_rate": 2.4654059040590407e-06,
+      "loss": 1.3975,
+      "step": 1356
+    },
+    {
+      "epoch": 1.6888612321095209,
+      "grad_norm": 2.835801362991333,
+      "learning_rate": 2.46309963099631e-06,
+      "loss": 1.3421,
+      "step": 1357
+    },
+    {
+      "epoch": 1.6901057871810827,
+      "grad_norm": 2.9438889026641846,
+      "learning_rate": 2.4607933579335795e-06,
+      "loss": 1.4422,
+      "step": 1358
+    },
+    {
+      "epoch": 1.6913503422526448,
+      "grad_norm": 3.168829917907715,
+      "learning_rate": 2.458487084870849e-06,
+      "loss": 1.4596,
+      "step": 1359
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "grad_norm": 2.8412039279937744,
+      "learning_rate": 2.4561808118081184e-06,
+      "loss": 1.4263,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "eval_loss": 1.6265885829925537,
+      "eval_runtime": 50.3402,
+      "eval_samples_per_second": 19.865,
+      "eval_steps_per_second": 0.834,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6938394523957685,
+      "grad_norm": 3.0540060997009277,
+      "learning_rate": 2.453874538745388e-06,
+      "loss": 1.4123,
+      "step": 1361
+    },
+    {
+      "epoch": 1.6950840074673303,
+      "grad_norm": 3.0544989109039307,
+      "learning_rate": 2.451568265682657e-06,
+      "loss": 1.412,
+      "step": 1362
+    },
+    {
+      "epoch": 1.6963285625388922,
+      "grad_norm": 3.114126205444336,
+      "learning_rate": 2.4492619926199267e-06,
+      "loss": 1.4952,
+      "step": 1363
+    },
+    {
+      "epoch": 1.6975731176104543,
+      "grad_norm": 2.9366016387939453,
+      "learning_rate": 2.4469557195571957e-06,
+      "loss": 1.5155,
+      "step": 1364
+    },
+    {
+      "epoch": 1.6988176726820163,
+      "grad_norm": 2.910677671432495,
+      "learning_rate": 2.444649446494465e-06,
+      "loss": 1.4393,
+      "step": 1365
+    },
+    {
+      "epoch": 1.7000622277535782,
+      "grad_norm": 2.8373727798461914,
+      "learning_rate": 2.4423431734317345e-06,
+      "loss": 1.4223,
+      "step": 1366
+    },
+    {
+      "epoch": 1.70130678282514,
+      "grad_norm": 2.865161895751953,
+      "learning_rate": 2.440036900369004e-06,
+      "loss": 1.4605,
+      "step": 1367
+    },
+    {
+      "epoch": 1.7025513378967019,
+      "grad_norm": 2.7929177284240723,
+      "learning_rate": 2.437730627306273e-06,
+      "loss": 1.4695,
+      "step": 1368
+    },
+    {
+      "epoch": 1.7037958929682637,
+      "grad_norm": 2.867953300476074,
+      "learning_rate": 2.4354243542435428e-06,
+      "loss": 1.3524,
+      "step": 1369
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "grad_norm": 2.8980207443237305,
+      "learning_rate": 2.4331180811808118e-06,
+      "loss": 1.4568,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "eval_loss": 1.6297556161880493,
+      "eval_runtime": 51.447,
+      "eval_samples_per_second": 19.437,
+      "eval_steps_per_second": 0.816,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7062850031113876,
+      "grad_norm": 2.954972505569458,
+      "learning_rate": 2.4308118081180816e-06,
+      "loss": 1.4,
+      "step": 1371
+    },
+    {
+      "epoch": 1.7075295581829497,
+      "grad_norm": 2.973191738128662,
+      "learning_rate": 2.4285055350553506e-06,
+      "loss": 1.4062,
+      "step": 1372
+    },
+    {
+      "epoch": 1.7087741132545116,
+      "grad_norm": 2.951444387435913,
+      "learning_rate": 2.42619926199262e-06,
+      "loss": 1.4523,
+      "step": 1373
+    },
+    {
+      "epoch": 1.7100186683260734,
+      "grad_norm": 2.850445508956909,
+      "learning_rate": 2.4238929889298895e-06,
+      "loss": 1.4282,
+      "step": 1374
+    },
+    {
+      "epoch": 1.7112632233976353,
+      "grad_norm": 2.7643561363220215,
+      "learning_rate": 2.421586715867159e-06,
+      "loss": 1.3807,
+      "step": 1375
+    },
+    {
+      "epoch": 1.712507778469197,
+      "grad_norm": 2.8872134685516357,
+      "learning_rate": 2.4192804428044283e-06,
+      "loss": 1.4801,
+      "step": 1376
+    },
+    {
+      "epoch": 1.7137523335407592,
+      "grad_norm": 2.8748483657836914,
+      "learning_rate": 2.4169741697416977e-06,
+      "loss": 1.381,
+      "step": 1377
+    },
+    {
+      "epoch": 1.7149968886123212,
+      "grad_norm": 2.7833831310272217,
+      "learning_rate": 2.4146678966789667e-06,
+      "loss": 1.389,
+      "step": 1378
+    },
+    {
+      "epoch": 1.716241443683883,
+      "grad_norm": 2.994715690612793,
+      "learning_rate": 2.4123616236162366e-06,
+      "loss": 1.3917,
+      "step": 1379
+    },
+    {
+      "epoch": 1.717485998755445,
+      "grad_norm": 2.887026786804199,
+      "learning_rate": 2.4100553505535056e-06,
+      "loss": 1.3725,
+      "step": 1380
+    },
+    {
+      "epoch": 1.717485998755445,
+      "eval_loss": 1.6283434629440308,
+      "eval_runtime": 50.596,
+      "eval_samples_per_second": 19.764,
+      "eval_steps_per_second": 0.83,
+      "step": 1380
+    },
+    {
+      "epoch": 1.7187305538270068,
+      "grad_norm": 3.2046546936035156,
+      "learning_rate": 2.407749077490775e-06,
+      "loss": 1.4422,
+      "step": 1381
+    },
+    {
+      "epoch": 1.7199751088985686,
+      "grad_norm": 3.0105698108673096,
+      "learning_rate": 2.4054428044280444e-06,
+      "loss": 1.4494,
+      "step": 1382
+    },
+    {
+      "epoch": 1.7212196639701307,
+      "grad_norm": 2.9917800426483154,
+      "learning_rate": 2.403136531365314e-06,
+      "loss": 1.4407,
+      "step": 1383
+    },
+    {
+      "epoch": 1.7224642190416926,
+      "grad_norm": 2.844325065612793,
+      "learning_rate": 2.4008302583025833e-06,
+      "loss": 1.4067,
+      "step": 1384
+    },
+    {
+      "epoch": 1.7237087741132546,
+      "grad_norm": 2.919576406478882,
+      "learning_rate": 2.3985239852398527e-06,
+      "loss": 1.4352,
+      "step": 1385
+    },
+    {
+      "epoch": 1.7249533291848165,
+      "grad_norm": 2.8357744216918945,
+      "learning_rate": 2.3962177121771217e-06,
+      "loss": 1.3978,
+      "step": 1386
+    },
+    {
+      "epoch": 1.7261978842563783,
+      "grad_norm": 2.82425856590271,
+      "learning_rate": 2.3939114391143915e-06,
+      "loss": 1.4439,
+      "step": 1387
+    },
+    {
+      "epoch": 1.7274424393279402,
+      "grad_norm": 2.9155638217926025,
+      "learning_rate": 2.3916051660516605e-06,
+      "loss": 1.4326,
+      "step": 1388
+    },
+    {
+      "epoch": 1.728686994399502,
+      "grad_norm": 2.862804651260376,
+      "learning_rate": 2.3892988929889304e-06,
+      "loss": 1.3766,
+      "step": 1389
+    },
+    {
+      "epoch": 1.729931549471064,
+      "grad_norm": 3.012230634689331,
+      "learning_rate": 2.3869926199261994e-06,
+      "loss": 1.4569,
+      "step": 1390
+    },
+    {
+      "epoch": 1.729931549471064,
+      "eval_loss": 1.6268510818481445,
+      "eval_runtime": 52.5438,
+      "eval_samples_per_second": 19.032,
+      "eval_steps_per_second": 0.799,
+      "step": 1390
+    },
+    {
+      "epoch": 1.7311761045426262,
+      "grad_norm": 2.9922425746917725,
+      "learning_rate": 2.384686346863469e-06,
+      "loss": 1.4066,
+      "step": 1391
+    },
+    {
+      "epoch": 1.732420659614188,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 2.3823800738007382e-06,
+      "loss": 1.4514,
+      "step": 1392
+    },
+    {
+      "epoch": 1.7336652146857499,
+      "grad_norm": 2.834998846054077,
+      "learning_rate": 2.3800738007380077e-06,
+      "loss": 1.4714,
+      "step": 1393
+    },
+    {
+      "epoch": 1.7349097697573117,
+      "grad_norm": 2.9646077156066895,
+      "learning_rate": 2.377767527675277e-06,
+      "loss": 1.4368,
+      "step": 1394
+    },
+    {
+      "epoch": 1.7361543248288736,
+      "grad_norm": 2.8233699798583984,
+      "learning_rate": 2.3754612546125465e-06,
+      "loss": 1.4598,
+      "step": 1395
+    },
+    {
+      "epoch": 1.7373988799004356,
+      "grad_norm": 2.895472764968872,
+      "learning_rate": 2.3731549815498155e-06,
+      "loss": 1.4551,
+      "step": 1396
+    },
+    {
+      "epoch": 1.7386434349719975,
+      "grad_norm": 2.7917752265930176,
+      "learning_rate": 2.3708487084870853e-06,
+      "loss": 1.4759,
+      "step": 1397
+    },
+    {
+      "epoch": 1.7398879900435595,
+      "grad_norm": 2.820672035217285,
+      "learning_rate": 2.3685424354243543e-06,
+      "loss": 1.3711,
+      "step": 1398
+    },
+    {
+      "epoch": 1.7411325451151214,
+      "grad_norm": 2.804952621459961,
+      "learning_rate": 2.3662361623616238e-06,
+      "loss": 1.4227,
+      "step": 1399
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "grad_norm": 2.981553316116333,
+      "learning_rate": 2.363929889298893e-06,
+      "loss": 1.4639,
+      "step": 1400
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "eval_loss": 1.6229957342147827,
+      "eval_runtime": 49.1507,
+      "eval_samples_per_second": 20.346,
+      "eval_steps_per_second": 0.855,
+      "step": 1400
+    },
+    {
+      "epoch": 1.743621655258245,
+      "grad_norm": 2.8649392127990723,
+      "learning_rate": 2.3616236162361626e-06,
+      "loss": 1.4283,
+      "step": 1401
+    },
+    {
+      "epoch": 1.744866210329807,
+      "grad_norm": 2.9438977241516113,
+      "learning_rate": 2.359317343173432e-06,
+      "loss": 1.444,
+      "step": 1402
+    },
+    {
+      "epoch": 1.746110765401369,
+      "grad_norm": 2.9270176887512207,
+      "learning_rate": 2.3570110701107015e-06,
+      "loss": 1.4044,
+      "step": 1403
+    },
+    {
+      "epoch": 1.747355320472931,
+      "grad_norm": 2.791527271270752,
+      "learning_rate": 2.3547047970479705e-06,
+      "loss": 1.4195,
+      "step": 1404
+    },
+    {
+      "epoch": 1.748599875544493,
+      "grad_norm": 2.8519608974456787,
+      "learning_rate": 2.3523985239852403e-06,
+      "loss": 1.4031,
+      "step": 1405
+    },
+    {
+      "epoch": 1.7498444306160548,
+      "grad_norm": 2.90872859954834,
+      "learning_rate": 2.3500922509225093e-06,
+      "loss": 1.4616,
+      "step": 1406
+    },
+    {
+      "epoch": 1.7510889856876166,
+      "grad_norm": 2.9190874099731445,
+      "learning_rate": 2.3477859778597787e-06,
+      "loss": 1.3917,
+      "step": 1407
+    },
+    {
+      "epoch": 1.7523335407591785,
+      "grad_norm": 2.868323802947998,
+      "learning_rate": 2.345479704797048e-06,
+      "loss": 1.4053,
+      "step": 1408
+    },
+    {
+      "epoch": 1.7535780958307405,
+      "grad_norm": 2.8638579845428467,
+      "learning_rate": 2.3431734317343176e-06,
+      "loss": 1.4042,
+      "step": 1409
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "grad_norm": 3.1759817600250244,
+      "learning_rate": 2.340867158671587e-06,
+      "loss": 1.4937,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "eval_loss": 1.6252617835998535,
+      "eval_runtime": 44.3041,
+      "eval_samples_per_second": 22.571,
+      "eval_steps_per_second": 0.948,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7560672059738645,
+      "grad_norm": 2.8265459537506104,
+      "learning_rate": 2.3385608856088564e-06,
+      "loss": 1.406,
+      "step": 1411
+    },
+    {
+      "epoch": 1.7573117610454263,
+      "grad_norm": 2.856229305267334,
+      "learning_rate": 2.336254612546126e-06,
+      "loss": 1.4395,
+      "step": 1412
+    },
+    {
+      "epoch": 1.7585563161169881,
+      "grad_norm": 2.9112775325775146,
+      "learning_rate": 2.333948339483395e-06,
+      "loss": 1.427,
+      "step": 1413
+    },
+    {
+      "epoch": 1.75980087118855,
+      "grad_norm": 3.098846435546875,
+      "learning_rate": 2.3316420664206643e-06,
+      "loss": 1.4222,
+      "step": 1414
+    },
+    {
+      "epoch": 1.7610454262601118,
+      "grad_norm": 2.758100986480713,
+      "learning_rate": 2.3293357933579337e-06,
+      "loss": 1.3844,
+      "step": 1415
+    },
+    {
+      "epoch": 1.762289981331674,
+      "grad_norm": 2.8422274589538574,
+      "learning_rate": 2.327029520295203e-06,
+      "loss": 1.4033,
+      "step": 1416
+    },
+    {
+      "epoch": 1.763534536403236,
+      "grad_norm": 2.9257912635803223,
+      "learning_rate": 2.3247232472324725e-06,
+      "loss": 1.3961,
+      "step": 1417
+    },
+    {
+      "epoch": 1.7647790914747978,
+      "grad_norm": 2.9431657791137695,
+      "learning_rate": 2.322416974169742e-06,
+      "loss": 1.4451,
+      "step": 1418
+    },
+    {
+      "epoch": 1.7660236465463597,
+      "grad_norm": 2.8598814010620117,
+      "learning_rate": 2.3201107011070114e-06,
+      "loss": 1.3816,
+      "step": 1419
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "grad_norm": 2.9868006706237793,
+      "learning_rate": 2.317804428044281e-06,
+      "loss": 1.4037,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "eval_loss": 1.6231273412704468,
+      "eval_runtime": 48.8067,
+      "eval_samples_per_second": 20.489,
+      "eval_steps_per_second": 0.861,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7685127566894834,
+      "grad_norm": 2.888206720352173,
+      "learning_rate": 2.31549815498155e-06,
+      "loss": 1.4087,
+      "step": 1421
+    },
+    {
+      "epoch": 1.7697573117610454,
+      "grad_norm": 2.9240078926086426,
+      "learning_rate": 2.3131918819188192e-06,
+      "loss": 1.3909,
+      "step": 1422
+    },
+    {
+      "epoch": 1.7710018668326073,
+      "grad_norm": 3.0401012897491455,
+      "learning_rate": 2.3108856088560886e-06,
+      "loss": 1.423,
+      "step": 1423
+    },
+    {
+      "epoch": 1.7722464219041694,
+      "grad_norm": 2.9484200477600098,
+      "learning_rate": 2.308579335793358e-06,
+      "loss": 1.449,
+      "step": 1424
+    },
+    {
+      "epoch": 1.7734909769757312,
+      "grad_norm": 2.863267660140991,
+      "learning_rate": 2.3062730627306275e-06,
+      "loss": 1.4215,
+      "step": 1425
+    },
+    {
+      "epoch": 1.774735532047293,
+      "grad_norm": 2.9615612030029297,
+      "learning_rate": 2.303966789667897e-06,
+      "loss": 1.4391,
+      "step": 1426
+    },
+    {
+      "epoch": 1.775980087118855,
+      "grad_norm": 2.7958569526672363,
+      "learning_rate": 2.3016605166051663e-06,
+      "loss": 1.3662,
+      "step": 1427
+    },
+    {
+      "epoch": 1.777224642190417,
+      "grad_norm": 2.898787021636963,
+      "learning_rate": 2.2993542435424358e-06,
+      "loss": 1.362,
+      "step": 1428
+    },
+    {
+      "epoch": 1.7784691972619788,
+      "grad_norm": 2.883049726486206,
+      "learning_rate": 2.2970479704797048e-06,
+      "loss": 1.4339,
+      "step": 1429
+    },
+    {
+      "epoch": 1.779713752333541,
+      "grad_norm": 3.0204827785491943,
+      "learning_rate": 2.294741697416974e-06,
+      "loss": 1.5174,
+      "step": 1430
+    },
+    {
+      "epoch": 1.779713752333541,
+      "eval_loss": 1.6246228218078613,
+      "eval_runtime": 47.5611,
+      "eval_samples_per_second": 21.026,
+      "eval_steps_per_second": 0.883,
+      "step": 1430
+    },
+    {
+      "epoch": 1.7809583074051027,
+      "grad_norm": 2.9354310035705566,
+      "learning_rate": 2.2924354243542436e-06,
+      "loss": 1.4115,
+      "step": 1431
+    },
+    {
+      "epoch": 1.7822028624766646,
+      "grad_norm": 2.9498608112335205,
+      "learning_rate": 2.290129151291513e-06,
+      "loss": 1.4058,
+      "step": 1432
+    },
+    {
+      "epoch": 1.7834474175482264,
+      "grad_norm": 2.9706921577453613,
+      "learning_rate": 2.2878228782287825e-06,
+      "loss": 1.426,
+      "step": 1433
+    },
+    {
+      "epoch": 1.7846919726197883,
+      "grad_norm": 2.8945024013519287,
+      "learning_rate": 2.285516605166052e-06,
+      "loss": 1.4011,
+      "step": 1434
+    },
+    {
+      "epoch": 1.7859365276913504,
+      "grad_norm": 2.8987081050872803,
+      "learning_rate": 2.2832103321033213e-06,
+      "loss": 1.4163,
+      "step": 1435
+    },
+    {
+      "epoch": 1.7871810827629122,
+      "grad_norm": 2.9797592163085938,
+      "learning_rate": 2.2809040590405907e-06,
+      "loss": 1.4367,
+      "step": 1436
+    },
+    {
+      "epoch": 1.7884256378344743,
+      "grad_norm": 2.8872063159942627,
+      "learning_rate": 2.2785977859778597e-06,
+      "loss": 1.4226,
+      "step": 1437
+    },
+    {
+      "epoch": 1.7896701929060361,
+      "grad_norm": 2.942655563354492,
+      "learning_rate": 2.2762915129151296e-06,
+      "loss": 1.4132,
+      "step": 1438
+    },
+    {
+      "epoch": 1.790914747977598,
+      "grad_norm": 3.027392625808716,
+      "learning_rate": 2.2739852398523986e-06,
+      "loss": 1.4148,
+      "step": 1439
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "grad_norm": 2.9475181102752686,
+      "learning_rate": 2.271678966789668e-06,
+      "loss": 1.4585,
+      "step": 1440
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "eval_loss": 1.6234897375106812,
+      "eval_runtime": 53.3737,
+      "eval_samples_per_second": 18.736,
+      "eval_steps_per_second": 0.787,
+      "step": 1440
+    },
+    {
+      "epoch": 1.793403858120722,
+      "grad_norm": 2.846304416656494,
+      "learning_rate": 2.2693726937269374e-06,
+      "loss": 1.3667,
+      "step": 1441
+    },
+    {
+      "epoch": 1.7946484131922837,
+      "grad_norm": 2.8925700187683105,
+      "learning_rate": 2.267066420664207e-06,
+      "loss": 1.4318,
+      "step": 1442
+    },
+    {
+      "epoch": 1.7958929682638458,
+      "grad_norm": 2.884025812149048,
+      "learning_rate": 2.2647601476014763e-06,
+      "loss": 1.4395,
+      "step": 1443
+    },
+    {
+      "epoch": 1.7971375233354077,
+      "grad_norm": 2.9763753414154053,
+      "learning_rate": 2.2624538745387457e-06,
+      "loss": 1.3742,
+      "step": 1444
+    },
+    {
+      "epoch": 1.7983820784069695,
+      "grad_norm": 3.0834898948669434,
+      "learning_rate": 2.2601476014760147e-06,
+      "loss": 1.4651,
+      "step": 1445
+    },
+    {
+      "epoch": 1.7996266334785314,
+      "grad_norm": 3.0673978328704834,
+      "learning_rate": 2.2578413284132845e-06,
+      "loss": 1.475,
+      "step": 1446
+    },
+    {
+      "epoch": 1.8008711885500932,
+      "grad_norm": 2.931521415710449,
+      "learning_rate": 2.2555350553505535e-06,
+      "loss": 1.4506,
+      "step": 1447
+    },
+    {
+      "epoch": 1.8021157436216553,
+      "grad_norm": 2.953301429748535,
+      "learning_rate": 2.253228782287823e-06,
+      "loss": 1.4224,
+      "step": 1448
+    },
+    {
+      "epoch": 1.8033602986932171,
+      "grad_norm": 3.0210883617401123,
+      "learning_rate": 2.2509225092250924e-06,
+      "loss": 1.4387,
+      "step": 1449
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "grad_norm": 2.975741386413574,
+      "learning_rate": 2.248616236162362e-06,
+      "loss": 1.4318,
+      "step": 1450
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "eval_loss": 1.618930697441101,
+      "eval_runtime": 49.6692,
+      "eval_samples_per_second": 20.133,
+      "eval_steps_per_second": 0.846,
+      "step": 1450
+    },
+    {
+      "epoch": 1.805849408836341,
+      "grad_norm": 2.9083876609802246,
+      "learning_rate": 2.2463099630996312e-06,
+      "loss": 1.3837,
+      "step": 1451
+    },
+    {
+      "epoch": 1.807093963907903,
+      "grad_norm": 3.364248275756836,
+      "learning_rate": 2.2440036900369006e-06,
+      "loss": 1.4589,
+      "step": 1452
+    },
+    {
+      "epoch": 1.8083385189794647,
+      "grad_norm": 2.935478925704956,
+      "learning_rate": 2.2416974169741696e-06,
+      "loss": 1.4119,
+      "step": 1453
+    },
+    {
+      "epoch": 1.8095830740510268,
+      "grad_norm": 3.1742911338806152,
+      "learning_rate": 2.2393911439114395e-06,
+      "loss": 1.4203,
+      "step": 1454
+    },
+    {
+      "epoch": 1.8108276291225887,
+      "grad_norm": 2.96966290473938,
+      "learning_rate": 2.2370848708487085e-06,
+      "loss": 1.4054,
+      "step": 1455
+    },
+    {
+      "epoch": 1.8120721841941507,
+      "grad_norm": 3.0919055938720703,
+      "learning_rate": 2.2347785977859783e-06,
+      "loss": 1.4289,
+      "step": 1456
+    },
+    {
+      "epoch": 1.8133167392657126,
+      "grad_norm": 2.9061055183410645,
+      "learning_rate": 2.2324723247232473e-06,
+      "loss": 1.4958,
+      "step": 1457
+    },
+    {
+      "epoch": 1.8145612943372744,
+      "grad_norm": 2.968348503112793,
+      "learning_rate": 2.2301660516605168e-06,
+      "loss": 1.4033,
+      "step": 1458
+    },
+    {
+      "epoch": 1.8158058494088363,
+      "grad_norm": 3.0435378551483154,
+      "learning_rate": 2.227859778597786e-06,
+      "loss": 1.414,
+      "step": 1459
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "grad_norm": 2.9067916870117188,
+      "learning_rate": 2.2255535055350556e-06,
+      "loss": 1.3954,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "eval_loss": 1.6240814924240112,
+      "eval_runtime": 51.7164,
+      "eval_samples_per_second": 19.336,
+      "eval_steps_per_second": 0.812,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8182949595519602,
+      "grad_norm": 3.156384229660034,
+      "learning_rate": 2.2232472324723246e-06,
+      "loss": 1.4658,
+      "step": 1461
+    },
+    {
+      "epoch": 1.819539514623522,
+      "grad_norm": 2.9885313510894775,
+      "learning_rate": 2.2209409594095944e-06,
+      "loss": 1.4123,
+      "step": 1462
+    },
+    {
+      "epoch": 1.820784069695084,
+      "grad_norm": 3.227780818939209,
+      "learning_rate": 2.2186346863468634e-06,
+      "loss": 1.3587,
+      "step": 1463
+    },
+    {
+      "epoch": 1.822028624766646,
+      "grad_norm": 3.0375192165374756,
+      "learning_rate": 2.2163284132841333e-06,
+      "loss": 1.4015,
+      "step": 1464
+    },
+    {
+      "epoch": 1.8232731798382078,
+      "grad_norm": 3.0200846195220947,
+      "learning_rate": 2.2140221402214023e-06,
+      "loss": 1.419,
+      "step": 1465
+    },
+    {
+      "epoch": 1.8245177349097697,
+      "grad_norm": 2.9456615447998047,
+      "learning_rate": 2.2117158671586717e-06,
+      "loss": 1.4075,
+      "step": 1466
+    },
+    {
+      "epoch": 1.8257622899813317,
+      "grad_norm": 3.3129165172576904,
+      "learning_rate": 2.209409594095941e-06,
+      "loss": 1.4241,
+      "step": 1467
+    },
+    {
+      "epoch": 1.8270068450528936,
+      "grad_norm": 3.013141393661499,
+      "learning_rate": 2.2071033210332106e-06,
+      "loss": 1.4066,
+      "step": 1468
+    },
+    {
+      "epoch": 1.8282514001244556,
+      "grad_norm": 2.8414366245269775,
+      "learning_rate": 2.20479704797048e-06,
+      "loss": 1.4009,
+      "step": 1469
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "grad_norm": 2.8920438289642334,
+      "learning_rate": 2.2024907749077494e-06,
+      "loss": 1.3916,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "eval_loss": 1.6220412254333496,
+      "eval_runtime": 52.6199,
+      "eval_samples_per_second": 19.004,
+      "eval_steps_per_second": 0.798,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8307405102675793,
+      "grad_norm": 3.032353639602661,
+      "learning_rate": 2.2001845018450184e-06,
+      "loss": 1.402,
+      "step": 1471
+    },
+    {
+      "epoch": 1.8319850653391412,
+      "grad_norm": 2.9512081146240234,
+      "learning_rate": 2.1978782287822883e-06,
+      "loss": 1.429,
+      "step": 1472
+    },
+    {
+      "epoch": 1.833229620410703,
+      "grad_norm": 2.846081018447876,
+      "learning_rate": 2.1955719557195573e-06,
+      "loss": 1.4203,
+      "step": 1473
+    },
+    {
+      "epoch": 1.834474175482265,
+      "grad_norm": 3.093378782272339,
+      "learning_rate": 2.1932656826568267e-06,
+      "loss": 1.4307,
+      "step": 1474
+    },
+    {
+      "epoch": 1.8357187305538272,
+      "grad_norm": 2.808326005935669,
+      "learning_rate": 2.190959409594096e-06,
+      "loss": 1.3753,
+      "step": 1475
+    },
+    {
+      "epoch": 1.836963285625389,
+      "grad_norm": 3.0328383445739746,
+      "learning_rate": 2.1886531365313655e-06,
+      "loss": 1.4534,
+      "step": 1476
+    },
+    {
+      "epoch": 1.8382078406969509,
+      "grad_norm": 3.045897960662842,
+      "learning_rate": 2.186346863468635e-06,
+      "loss": 1.3259,
+      "step": 1477
+    },
+    {
+      "epoch": 1.8394523957685127,
+      "grad_norm": 3.171846628189087,
+      "learning_rate": 2.1840405904059044e-06,
+      "loss": 1.397,
+      "step": 1478
+    },
+    {
+      "epoch": 1.8406969508400746,
+      "grad_norm": 2.893061637878418,
+      "learning_rate": 2.1817343173431734e-06,
+      "loss": 1.4201,
+      "step": 1479
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "grad_norm": 3.0151467323303223,
+      "learning_rate": 2.1794280442804432e-06,
+      "loss": 1.4266,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "eval_loss": 1.6166906356811523,
+      "eval_runtime": 51.8266,
+      "eval_samples_per_second": 19.295,
+      "eval_steps_per_second": 0.81,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8431860609831985,
+      "grad_norm": 2.942431688308716,
+      "learning_rate": 2.177121771217712e-06,
+      "loss": 1.3744,
+      "step": 1481
+    },
+    {
+      "epoch": 1.8444306160547606,
+      "grad_norm": 2.8482232093811035,
+      "learning_rate": 2.1748154981549816e-06,
+      "loss": 1.3914,
+      "step": 1482
+    },
+    {
+      "epoch": 1.8456751711263224,
+      "grad_norm": 2.9315359592437744,
+      "learning_rate": 2.172509225092251e-06,
+      "loss": 1.419,
+      "step": 1483
+    },
+    {
+      "epoch": 1.8469197261978842,
+      "grad_norm": 3.191800594329834,
+      "learning_rate": 2.1702029520295205e-06,
+      "loss": 1.3524,
+      "step": 1484
+    },
+    {
+      "epoch": 1.848164281269446,
+      "grad_norm": 2.9308040142059326,
+      "learning_rate": 2.16789667896679e-06,
+      "loss": 1.3742,
+      "step": 1485
+    },
+    {
+      "epoch": 1.849408836341008,
+      "grad_norm": 2.7678964138031006,
+      "learning_rate": 2.1655904059040593e-06,
+      "loss": 1.3786,
+      "step": 1486
+    },
+    {
+      "epoch": 1.85065339141257,
+      "grad_norm": 3.0300636291503906,
+      "learning_rate": 2.1632841328413287e-06,
+      "loss": 1.4469,
+      "step": 1487
+    },
+    {
+      "epoch": 1.851897946484132,
+      "grad_norm": 3.142781972885132,
+      "learning_rate": 2.160977859778598e-06,
+      "loss": 1.4565,
+      "step": 1488
+    },
+    {
+      "epoch": 1.853142501555694,
+      "grad_norm": 2.954916000366211,
+      "learning_rate": 2.158671586715867e-06,
+      "loss": 1.4356,
+      "step": 1489
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "grad_norm": 2.864875316619873,
+      "learning_rate": 2.1563653136531366e-06,
+      "loss": 1.4521,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "eval_loss": 1.6207302808761597,
+      "eval_runtime": 49.2215,
+      "eval_samples_per_second": 20.316,
+      "eval_steps_per_second": 0.853,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8556316116988176,
+      "grad_norm": 2.8908467292785645,
+      "learning_rate": 2.154059040590406e-06,
+      "loss": 1.4304,
+      "step": 1491
+    },
+    {
+      "epoch": 1.8568761667703795,
+      "grad_norm": 2.9439549446105957,
+      "learning_rate": 2.1517527675276754e-06,
+      "loss": 1.398,
+      "step": 1492
+    },
+    {
+      "epoch": 1.8581207218419415,
+      "grad_norm": 2.967728853225708,
+      "learning_rate": 2.149446494464945e-06,
+      "loss": 1.5038,
+      "step": 1493
+    },
+    {
+      "epoch": 1.8593652769135034,
+      "grad_norm": 2.938123941421509,
+      "learning_rate": 2.1471402214022143e-06,
+      "loss": 1.4395,
+      "step": 1494
+    },
+    {
+      "epoch": 1.8606098319850655,
+      "grad_norm": 2.93609881401062,
+      "learning_rate": 2.1448339483394837e-06,
+      "loss": 1.3939,
+      "step": 1495
+    },
+    {
+      "epoch": 1.8618543870566273,
+      "grad_norm": 2.9093775749206543,
+      "learning_rate": 2.142527675276753e-06,
+      "loss": 1.3942,
+      "step": 1496
+    },
+    {
+      "epoch": 1.8630989421281892,
+      "grad_norm": 2.773960590362549,
+      "learning_rate": 2.140221402214022e-06,
+      "loss": 1.3886,
+      "step": 1497
+    },
+    {
+      "epoch": 1.864343497199751,
+      "grad_norm": 2.9858205318450928,
+      "learning_rate": 2.1379151291512916e-06,
+      "loss": 1.3814,
+      "step": 1498
+    },
+    {
+      "epoch": 1.8655880522713129,
+      "grad_norm": 3.1233417987823486,
+      "learning_rate": 2.135608856088561e-06,
+      "loss": 1.4313,
+      "step": 1499
+    },
+    {
+      "epoch": 1.866832607342875,
+      "grad_norm": 2.9569480419158936,
+      "learning_rate": 2.1333025830258304e-06,
+      "loss": 1.4302,
+      "step": 1500
+    },
+    {
+      "epoch": 1.866832607342875,
+      "eval_loss": 1.6130231618881226,
+      "eval_runtime": 46.2834,
+      "eval_samples_per_second": 21.606,
+      "eval_steps_per_second": 0.907,
+      "step": 1500
+    },
+    {
+      "epoch": 1.868077162414437,
+      "grad_norm": 2.926203727722168,
+      "learning_rate": 2.1309963099631e-06,
+      "loss": 1.4236,
+      "step": 1501
+    },
+    {
+      "epoch": 1.8693217174859988,
+      "grad_norm": 2.9496357440948486,
+      "learning_rate": 2.1286900369003692e-06,
+      "loss": 1.4014,
+      "step": 1502
+    },
+    {
+      "epoch": 1.8705662725575607,
+      "grad_norm": 2.9314279556274414,
+      "learning_rate": 2.1263837638376387e-06,
+      "loss": 1.4414,
+      "step": 1503
+    },
+    {
+      "epoch": 1.8718108276291225,
+      "grad_norm": 2.889460802078247,
+      "learning_rate": 2.124077490774908e-06,
+      "loss": 1.3883,
+      "step": 1504
+    },
+    {
+      "epoch": 1.8730553827006844,
+      "grad_norm": 2.9859671592712402,
+      "learning_rate": 2.1217712177121775e-06,
+      "loss": 1.4551,
+      "step": 1505
+    },
+    {
+      "epoch": 1.8742999377722465,
+      "grad_norm": 2.9330570697784424,
+      "learning_rate": 2.1194649446494465e-06,
+      "loss": 1.3895,
+      "step": 1506
+    },
+    {
+      "epoch": 1.8755444928438083,
+      "grad_norm": 2.9062387943267822,
+      "learning_rate": 2.117158671586716e-06,
+      "loss": 1.4492,
+      "step": 1507
+    },
+    {
+      "epoch": 1.8767890479153704,
+      "grad_norm": 3.0337791442871094,
+      "learning_rate": 2.1148523985239854e-06,
+      "loss": 1.4682,
+      "step": 1508
+    },
+    {
+      "epoch": 1.8780336029869322,
+      "grad_norm": 3.0448431968688965,
+      "learning_rate": 2.1125461254612548e-06,
+      "loss": 1.4836,
+      "step": 1509
+    },
+    {
+      "epoch": 1.879278158058494,
+      "grad_norm": 2.912764310836792,
+      "learning_rate": 2.110239852398524e-06,
+      "loss": 1.4644,
+      "step": 1510
+    },
+    {
+      "epoch": 1.879278158058494,
+      "eval_loss": 1.6124104261398315,
+      "eval_runtime": 41.9713,
+      "eval_samples_per_second": 23.826,
+      "eval_steps_per_second": 1.001,
+      "step": 1510
+    },
+    {
+      "epoch": 1.880522713130056,
+      "grad_norm": 3.0399203300476074,
+      "learning_rate": 2.1079335793357936e-06,
+      "loss": 1.4391,
+      "step": 1511
+    },
+    {
+      "epoch": 1.8817672682016178,
+      "grad_norm": 2.9852235317230225,
+      "learning_rate": 2.1056273062730626e-06,
+      "loss": 1.346,
+      "step": 1512
+    },
+    {
+      "epoch": 1.8830118232731798,
+      "grad_norm": 2.904437780380249,
+      "learning_rate": 2.1033210332103325e-06,
+      "loss": 1.4523,
+      "step": 1513
+    },
+    {
+      "epoch": 1.884256378344742,
+      "grad_norm": 2.8988049030303955,
+      "learning_rate": 2.1010147601476015e-06,
+      "loss": 1.4001,
+      "step": 1514
+    },
+    {
+      "epoch": 1.8855009334163038,
+      "grad_norm": 2.9890260696411133,
+      "learning_rate": 2.098708487084871e-06,
+      "loss": 1.4074,
+      "step": 1515
+    },
+    {
+      "epoch": 1.8867454884878656,
+      "grad_norm": 3.0486819744110107,
+      "learning_rate": 2.0964022140221403e-06,
+      "loss": 1.461,
+      "step": 1516
+    },
+    {
+      "epoch": 1.8879900435594275,
+      "grad_norm": 2.8746016025543213,
+      "learning_rate": 2.0940959409594097e-06,
+      "loss": 1.4013,
+      "step": 1517
+    },
+    {
+      "epoch": 1.8892345986309893,
+      "grad_norm": 2.979480266571045,
+      "learning_rate": 2.091789667896679e-06,
+      "loss": 1.4567,
+      "step": 1518
+    },
+    {
+      "epoch": 1.8904791537025514,
+      "grad_norm": 2.8351497650146484,
+      "learning_rate": 2.0894833948339486e-06,
+      "loss": 1.3655,
+      "step": 1519
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "grad_norm": 2.9061646461486816,
+      "learning_rate": 2.0871771217712176e-06,
+      "loss": 1.4165,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "eval_loss": 1.6172568798065186,
+      "eval_runtime": 42.998,
+      "eval_samples_per_second": 23.257,
+      "eval_steps_per_second": 0.977,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8929682638456753,
+      "grad_norm": 2.8751838207244873,
+      "learning_rate": 2.0848708487084874e-06,
+      "loss": 1.4133,
+      "step": 1521
+    },
+    {
+      "epoch": 1.8942128189172371,
+      "grad_norm": 2.7513587474823,
+      "learning_rate": 2.0825645756457564e-06,
+      "loss": 1.3734,
+      "step": 1522
+    },
+    {
+      "epoch": 1.895457373988799,
+      "grad_norm": 2.9244418144226074,
+      "learning_rate": 2.0802583025830263e-06,
+      "loss": 1.4181,
+      "step": 1523
+    },
+    {
+      "epoch": 1.8967019290603608,
+      "grad_norm": 2.877495765686035,
+      "learning_rate": 2.0779520295202953e-06,
+      "loss": 1.4009,
+      "step": 1524
+    },
+    {
+      "epoch": 1.8979464841319227,
+      "grad_norm": 2.9316184520721436,
+      "learning_rate": 2.0756457564575647e-06,
+      "loss": 1.3796,
+      "step": 1525
+    },
+    {
+      "epoch": 1.8991910392034848,
+      "grad_norm": 3.0403976440429688,
+      "learning_rate": 2.073339483394834e-06,
+      "loss": 1.4101,
+      "step": 1526
+    },
+    {
+      "epoch": 1.9004355942750468,
+      "grad_norm": 3.0020930767059326,
+      "learning_rate": 2.0710332103321035e-06,
+      "loss": 1.4166,
+      "step": 1527
+    },
+    {
+      "epoch": 1.9016801493466087,
+      "grad_norm": 3.0937304496765137,
+      "learning_rate": 2.0687269372693725e-06,
+      "loss": 1.4599,
+      "step": 1528
+    },
+    {
+      "epoch": 1.9029247044181705,
+      "grad_norm": 2.901540517807007,
+      "learning_rate": 2.0664206642066424e-06,
+      "loss": 1.3642,
+      "step": 1529
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "grad_norm": 3.0517938137054443,
+      "learning_rate": 2.0641143911439114e-06,
+      "loss": 1.451,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "eval_loss": 1.6178728342056274,
+      "eval_runtime": 43.9067,
+      "eval_samples_per_second": 22.776,
+      "eval_steps_per_second": 0.957,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9054138145612942,
+      "grad_norm": 3.12484073638916,
+      "learning_rate": 2.0618081180811812e-06,
+      "loss": 1.4522,
+      "step": 1531
+    },
+    {
+      "epoch": 1.9066583696328563,
+      "grad_norm": 2.9447414875030518,
+      "learning_rate": 2.0595018450184502e-06,
+      "loss": 1.4154,
+      "step": 1532
+    },
+    {
+      "epoch": 1.9079029247044181,
+      "grad_norm": 2.909738779067993,
+      "learning_rate": 2.0571955719557197e-06,
+      "loss": 1.4133,
+      "step": 1533
+    },
+    {
+      "epoch": 1.9091474797759802,
+      "grad_norm": 3.1522421836853027,
+      "learning_rate": 2.054889298892989e-06,
+      "loss": 1.4777,
+      "step": 1534
+    },
+    {
+      "epoch": 1.910392034847542,
+      "grad_norm": 3.041447639465332,
+      "learning_rate": 2.0525830258302585e-06,
+      "loss": 1.4104,
+      "step": 1535
+    },
+    {
+      "epoch": 1.911636589919104,
+      "grad_norm": 3.01129412651062,
+      "learning_rate": 2.050276752767528e-06,
+      "loss": 1.3852,
+      "step": 1536
+    },
+    {
+      "epoch": 1.9128811449906657,
+      "grad_norm": 3.0697543621063232,
+      "learning_rate": 2.0479704797047974e-06,
+      "loss": 1.4389,
+      "step": 1537
+    },
+    {
+      "epoch": 1.9141257000622276,
+      "grad_norm": 2.9391236305236816,
+      "learning_rate": 2.0456642066420664e-06,
+      "loss": 1.4239,
+      "step": 1538
+    },
+    {
+      "epoch": 1.9153702551337897,
+      "grad_norm": 2.978236675262451,
+      "learning_rate": 2.043357933579336e-06,
+      "loss": 1.462,
+      "step": 1539
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "grad_norm": 2.8124167919158936,
+      "learning_rate": 2.041051660516605e-06,
+      "loss": 1.3186,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "eval_loss": 1.6120468378067017,
+      "eval_runtime": 43.3787,
+      "eval_samples_per_second": 23.053,
+      "eval_steps_per_second": 0.968,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9178593652769136,
+      "grad_norm": 3.029939889907837,
+      "learning_rate": 2.0387453874538746e-06,
+      "loss": 1.4481,
+      "step": 1541
+    },
+    {
+      "epoch": 1.9191039203484754,
+      "grad_norm": 2.8749356269836426,
+      "learning_rate": 2.036439114391144e-06,
+      "loss": 1.3735,
+      "step": 1542
+    },
+    {
+      "epoch": 1.9203484754200373,
+      "grad_norm": 2.9181101322174072,
+      "learning_rate": 2.0341328413284135e-06,
+      "loss": 1.4085,
+      "step": 1543
+    },
+    {
+      "epoch": 1.9215930304915991,
+      "grad_norm": 2.936319589614868,
+      "learning_rate": 2.031826568265683e-06,
+      "loss": 1.3827,
+      "step": 1544
+    },
+    {
+      "epoch": 1.9228375855631612,
+      "grad_norm": 2.9286844730377197,
+      "learning_rate": 2.0295202952029523e-06,
+      "loss": 1.4563,
+      "step": 1545
+    },
+    {
+      "epoch": 1.924082140634723,
+      "grad_norm": 2.790752649307251,
+      "learning_rate": 2.0272140221402213e-06,
+      "loss": 1.3935,
+      "step": 1546
+    },
+    {
+      "epoch": 1.9253266957062851,
+      "grad_norm": 2.878164529800415,
+      "learning_rate": 2.024907749077491e-06,
+      "loss": 1.4159,
+      "step": 1547
+    },
+    {
+      "epoch": 1.926571250777847,
+      "grad_norm": 3.0583767890930176,
+      "learning_rate": 2.02260147601476e-06,
+      "loss": 1.4036,
+      "step": 1548
+    },
+    {
+      "epoch": 1.9278158058494088,
+      "grad_norm": 2.943686008453369,
+      "learning_rate": 2.02029520295203e-06,
+      "loss": 1.4178,
+      "step": 1549
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "grad_norm": 2.983553886413574,
+      "learning_rate": 2.017988929889299e-06,
+      "loss": 1.3959,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "eval_loss": 1.610864520072937,
+      "eval_runtime": 44.5262,
+      "eval_samples_per_second": 22.459,
+      "eval_steps_per_second": 0.943,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9303049159925325,
+      "grad_norm": 3.0323703289031982,
+      "learning_rate": 2.0156826568265684e-06,
+      "loss": 1.4362,
+      "step": 1551
+    },
+    {
+      "epoch": 1.9315494710640946,
+      "grad_norm": 3.036627769470215,
+      "learning_rate": 2.013376383763838e-06,
+      "loss": 1.4132,
+      "step": 1552
+    },
+    {
+      "epoch": 1.9327940261356567,
+      "grad_norm": 2.901071310043335,
+      "learning_rate": 2.0110701107011073e-06,
+      "loss": 1.3846,
+      "step": 1553
+    },
+    {
+      "epoch": 1.9340385812072185,
+      "grad_norm": 3.002803325653076,
+      "learning_rate": 2.0087638376383767e-06,
+      "loss": 1.4402,
+      "step": 1554
+    },
+    {
+      "epoch": 1.9352831362787803,
+      "grad_norm": 2.969228982925415,
+      "learning_rate": 2.006457564575646e-06,
+      "loss": 1.4387,
+      "step": 1555
+    },
+    {
+      "epoch": 1.9365276913503422,
+      "grad_norm": 2.9773998260498047,
+      "learning_rate": 2.004151291512915e-06,
+      "loss": 1.4149,
+      "step": 1556
+    },
+    {
+      "epoch": 1.937772246421904,
+      "grad_norm": 2.831655502319336,
+      "learning_rate": 2.0018450184501845e-06,
+      "loss": 1.3753,
+      "step": 1557
+    },
+    {
+      "epoch": 1.9390168014934661,
+      "grad_norm": 2.835994005203247,
+      "learning_rate": 1.999538745387454e-06,
+      "loss": 1.345,
+      "step": 1558
+    },
+    {
+      "epoch": 1.940261356565028,
+      "grad_norm": 3.0328211784362793,
+      "learning_rate": 1.9972324723247234e-06,
+      "loss": 1.3473,
+      "step": 1559
+    },
+    {
+      "epoch": 1.94150591163659,
+      "grad_norm": 2.9441378116607666,
+      "learning_rate": 1.994926199261993e-06,
+      "loss": 1.3913,
+      "step": 1560
+    },
+    {
+      "epoch": 1.94150591163659,
+      "eval_loss": 1.6193854808807373,
+      "eval_runtime": 44.6194,
+      "eval_samples_per_second": 22.412,
+      "eval_steps_per_second": 0.941,
+      "step": 1560
+    },
+    {
+      "epoch": 1.9427504667081519,
+      "grad_norm": 2.905486822128296,
+      "learning_rate": 1.9926199261992622e-06,
+      "loss": 1.4043,
+      "step": 1561
+    },
+    {
+      "epoch": 1.9439950217797137,
+      "grad_norm": 2.7758026123046875,
+      "learning_rate": 1.9903136531365317e-06,
+      "loss": 1.3974,
+      "step": 1562
+    },
+    {
+      "epoch": 1.9452395768512756,
+      "grad_norm": 2.7883355617523193,
+      "learning_rate": 1.988007380073801e-06,
+      "loss": 1.4246,
+      "step": 1563
+    },
+    {
+      "epoch": 1.9464841319228374,
+      "grad_norm": 2.8322808742523193,
+      "learning_rate": 1.98570110701107e-06,
+      "loss": 1.4002,
+      "step": 1564
+    },
+    {
+      "epoch": 1.9477286869943995,
+      "grad_norm": 2.948850631713867,
+      "learning_rate": 1.9833948339483395e-06,
+      "loss": 1.3866,
+      "step": 1565
+    },
+    {
+      "epoch": 1.9489732420659616,
+      "grad_norm": 2.9772586822509766,
+      "learning_rate": 1.981088560885609e-06,
+      "loss": 1.4473,
+      "step": 1566
+    },
+    {
+      "epoch": 1.9502177971375234,
+      "grad_norm": 2.9183740615844727,
+      "learning_rate": 1.9787822878228783e-06,
+      "loss": 1.3943,
+      "step": 1567
+    },
+    {
+      "epoch": 1.9514623522090853,
+      "grad_norm": 3.10422682762146,
+      "learning_rate": 1.9764760147601478e-06,
+      "loss": 1.465,
+      "step": 1568
+    },
+    {
+      "epoch": 1.952706907280647,
+      "grad_norm": 3.001533031463623,
+      "learning_rate": 1.974169741697417e-06,
+      "loss": 1.4133,
+      "step": 1569
+    },
+    {
+      "epoch": 1.953951462352209,
+      "grad_norm": 2.9521870613098145,
+      "learning_rate": 1.9718634686346866e-06,
+      "loss": 1.4652,
+      "step": 1570
+    },
+    {
+      "epoch": 1.953951462352209,
+      "eval_loss": 1.617977499961853,
+      "eval_runtime": 45.7189,
+      "eval_samples_per_second": 21.873,
+      "eval_steps_per_second": 0.919,
+      "step": 1570
+    },
+    {
+      "epoch": 1.955196017423771,
+      "grad_norm": 2.9504826068878174,
+      "learning_rate": 1.969557195571956e-06,
+      "loss": 1.4117,
+      "step": 1571
+    },
+    {
+      "epoch": 1.9564405724953329,
+      "grad_norm": 2.9383559226989746,
+      "learning_rate": 1.9672509225092255e-06,
+      "loss": 1.3439,
+      "step": 1572
+    },
+    {
+      "epoch": 1.957685127566895,
+      "grad_norm": 2.850644588470459,
+      "learning_rate": 1.9649446494464945e-06,
+      "loss": 1.3521,
+      "step": 1573
+    },
+    {
+      "epoch": 1.9589296826384568,
+      "grad_norm": 2.8787410259246826,
+      "learning_rate": 1.962638376383764e-06,
+      "loss": 1.4268,
+      "step": 1574
+    },
+    {
+      "epoch": 1.9601742377100186,
+      "grad_norm": 2.966080665588379,
+      "learning_rate": 1.9603321033210333e-06,
+      "loss": 1.3367,
+      "step": 1575
+    },
+    {
+      "epoch": 1.9614187927815805,
+      "grad_norm": 2.858222007751465,
+      "learning_rate": 1.9580258302583027e-06,
+      "loss": 1.3842,
+      "step": 1576
+    },
+    {
+      "epoch": 1.9626633478531423,
+      "grad_norm": 2.924525737762451,
+      "learning_rate": 1.955719557195572e-06,
+      "loss": 1.4561,
+      "step": 1577
+    },
+    {
+      "epoch": 1.9639079029247044,
+      "grad_norm": 2.9281182289123535,
+      "learning_rate": 1.9534132841328416e-06,
+      "loss": 1.4179,
+      "step": 1578
+    },
+    {
+      "epoch": 1.9651524579962665,
+      "grad_norm": 2.982023000717163,
+      "learning_rate": 1.951107011070111e-06,
+      "loss": 1.4413,
+      "step": 1579
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "grad_norm": 2.7912652492523193,
+      "learning_rate": 1.9488007380073804e-06,
+      "loss": 1.3577,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "eval_loss": 1.6062211990356445,
+      "eval_runtime": 46.7931,
+      "eval_samples_per_second": 21.371,
+      "eval_steps_per_second": 0.898,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9676415681393902,
+      "grad_norm": 2.9585583209991455,
+      "learning_rate": 1.9464944649446494e-06,
+      "loss": 1.4084,
+      "step": 1581
+    },
+    {
+      "epoch": 1.968886123210952,
+      "grad_norm": 2.879201889038086,
+      "learning_rate": 1.944188191881919e-06,
+      "loss": 1.4164,
+      "step": 1582
+    },
+    {
+      "epoch": 1.9701306782825139,
+      "grad_norm": 2.9510116577148438,
+      "learning_rate": 1.9418819188191883e-06,
+      "loss": 1.4211,
+      "step": 1583
+    },
+    {
+      "epoch": 1.971375233354076,
+      "grad_norm": 2.973844528198242,
+      "learning_rate": 1.9395756457564577e-06,
+      "loss": 1.4358,
+      "step": 1584
+    },
+    {
+      "epoch": 1.9726197884256378,
+      "grad_norm": 2.939131736755371,
+      "learning_rate": 1.937269372693727e-06,
+      "loss": 1.407,
+      "step": 1585
+    },
+    {
+      "epoch": 1.9738643434971999,
+      "grad_norm": 2.956646680831909,
+      "learning_rate": 1.9349630996309965e-06,
+      "loss": 1.3872,
+      "step": 1586
+    },
+    {
+      "epoch": 1.9751088985687617,
+      "grad_norm": 3.070354461669922,
+      "learning_rate": 1.932656826568266e-06,
+      "loss": 1.412,
+      "step": 1587
+    },
+    {
+      "epoch": 1.9763534536403236,
+      "grad_norm": 2.8143749237060547,
+      "learning_rate": 1.9303505535055354e-06,
+      "loss": 1.3522,
+      "step": 1588
+    },
+    {
+      "epoch": 1.9775980087118854,
+      "grad_norm": 2.8610002994537354,
+      "learning_rate": 1.9280442804428044e-06,
+      "loss": 1.3988,
+      "step": 1589
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "grad_norm": 2.951023817062378,
+      "learning_rate": 1.925738007380074e-06,
+      "loss": 1.3855,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "eval_loss": 1.613527536392212,
+      "eval_runtime": 46.5242,
+      "eval_samples_per_second": 21.494,
+      "eval_steps_per_second": 0.903,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9800871188550093,
+      "grad_norm": 2.9333510398864746,
+      "learning_rate": 1.9234317343173432e-06,
+      "loss": 1.4518,
+      "step": 1591
+    },
+    {
+      "epoch": 1.9813316739265714,
+      "grad_norm": 2.847015619277954,
+      "learning_rate": 1.9211254612546126e-06,
+      "loss": 1.3999,
+      "step": 1592
+    },
+    {
+      "epoch": 1.9825762289981332,
+      "grad_norm": 2.969320058822632,
+      "learning_rate": 1.918819188191882e-06,
+      "loss": 1.4241,
+      "step": 1593
+    },
+    {
+      "epoch": 1.983820784069695,
+      "grad_norm": 2.862797498703003,
+      "learning_rate": 1.9165129151291515e-06,
+      "loss": 1.3929,
+      "step": 1594
+    },
+    {
+      "epoch": 1.985065339141257,
+      "grad_norm": 2.90384840965271,
+      "learning_rate": 1.914206642066421e-06,
+      "loss": 1.4127,
+      "step": 1595
+    },
+    {
+      "epoch": 1.9863098942128188,
+      "grad_norm": 2.8059256076812744,
+      "learning_rate": 1.9119003690036903e-06,
+      "loss": 1.4155,
+      "step": 1596
+    },
+    {
+      "epoch": 1.9875544492843809,
+      "grad_norm": 2.9068455696105957,
+      "learning_rate": 1.9095940959409593e-06,
+      "loss": 1.3924,
+      "step": 1597
+    },
+    {
+      "epoch": 1.9887990043559427,
+      "grad_norm": 2.903542995452881,
+      "learning_rate": 1.907287822878229e-06,
+      "loss": 1.4039,
+      "step": 1598
+    },
+    {
+      "epoch": 1.9900435594275048,
+      "grad_norm": 2.8566174507141113,
+      "learning_rate": 1.9049815498154984e-06,
+      "loss": 1.4276,
+      "step": 1599
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "grad_norm": 2.8891515731811523,
+      "learning_rate": 1.9026752767527676e-06,
+      "loss": 1.4555,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "eval_loss": 1.6086686849594116,
+      "eval_runtime": 46.2785,
+      "eval_samples_per_second": 21.608,
+      "eval_steps_per_second": 0.908,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9925326695706285,
+      "grad_norm": 2.933851957321167,
+      "learning_rate": 1.900369003690037e-06,
+      "loss": 1.3881,
+      "step": 1601
+    },
+    {
+      "epoch": 1.9937772246421903,
+      "grad_norm": 2.8393521308898926,
+      "learning_rate": 1.8980627306273065e-06,
+      "loss": 1.3959,
+      "step": 1602
+    },
+    {
+      "epoch": 1.9950217797137524,
+      "grad_norm": 2.839740753173828,
+      "learning_rate": 1.8957564575645759e-06,
+      "loss": 1.384,
+      "step": 1603
+    },
+    {
+      "epoch": 1.9962663347853142,
+      "grad_norm": 2.7918097972869873,
+      "learning_rate": 1.8934501845018453e-06,
+      "loss": 1.4173,
+      "step": 1604
+    },
+    {
+      "epoch": 1.9975108898568763,
+      "grad_norm": 2.920839548110962,
+      "learning_rate": 1.8911439114391145e-06,
+      "loss": 1.3693,
+      "step": 1605
+    },
+    {
+      "epoch": 1.9987554449284382,
+      "grad_norm": 2.9545562267303467,
+      "learning_rate": 1.888837638376384e-06,
+      "loss": 1.4062,
+      "step": 1606
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 2.912815570831299,
+      "learning_rate": 1.8865313653136534e-06,
+      "loss": 1.4146,
+      "step": 1607
+    },
+    {
+      "epoch": 2.001244555071562,
+      "grad_norm": 3.1125903129577637,
+      "learning_rate": 1.8842250922509226e-06,
+      "loss": 1.3294,
+      "step": 1608
+    },
+    {
+      "epoch": 2.0024891101431237,
+      "grad_norm": 3.123905658721924,
+      "learning_rate": 1.881918819188192e-06,
+      "loss": 1.4175,
+      "step": 1609
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "grad_norm": 3.055100440979004,
+      "learning_rate": 1.8796125461254614e-06,
+      "loss": 1.3628,
+      "step": 1610
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "eval_loss": 1.6198724508285522,
+      "eval_runtime": 42.5155,
+      "eval_samples_per_second": 23.521,
+      "eval_steps_per_second": 0.988,
+      "step": 1610
+    },
+    {
+      "epoch": 2.004978220286248,
+      "grad_norm": 3.087313413619995,
+      "learning_rate": 1.8773062730627308e-06,
+      "loss": 1.292,
+      "step": 1611
+    },
+    {
+      "epoch": 2.0062227753578097,
+      "grad_norm": 3.1849849224090576,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 1.3393,
+      "step": 1612
+    },
+    {
+      "epoch": 2.0074673304293715,
+      "grad_norm": 3.243750810623169,
+      "learning_rate": 1.8726937269372695e-06,
+      "loss": 1.3367,
+      "step": 1613
+    },
+    {
+      "epoch": 2.0087118855009334,
+      "grad_norm": 3.3086085319519043,
+      "learning_rate": 1.8703874538745389e-06,
+      "loss": 1.3099,
+      "step": 1614
+    },
+    {
+      "epoch": 2.0099564405724952,
+      "grad_norm": 3.388633966445923,
+      "learning_rate": 1.8680811808118083e-06,
+      "loss": 1.3816,
+      "step": 1615
+    },
+    {
+      "epoch": 2.011200995644057,
+      "grad_norm": 3.422671318054199,
+      "learning_rate": 1.8657749077490777e-06,
+      "loss": 1.3799,
+      "step": 1616
+    },
+    {
+      "epoch": 2.0124455507156194,
+      "grad_norm": 3.1104466915130615,
+      "learning_rate": 1.863468634686347e-06,
+      "loss": 1.3752,
+      "step": 1617
+    },
+    {
+      "epoch": 2.013690105787181,
+      "grad_norm": 3.2728488445281982,
+      "learning_rate": 1.8611623616236164e-06,
+      "loss": 1.322,
+      "step": 1618
+    },
+    {
+      "epoch": 2.014934660858743,
+      "grad_norm": 3.3357644081115723,
+      "learning_rate": 1.8588560885608858e-06,
+      "loss": 1.3193,
+      "step": 1619
+    },
+    {
+      "epoch": 2.016179215930305,
+      "grad_norm": 3.1026456356048584,
+      "learning_rate": 1.856549815498155e-06,
+      "loss": 1.3394,
+      "step": 1620
+    },
+    {
+      "epoch": 2.016179215930305,
+      "eval_loss": 1.6217585802078247,
+      "eval_runtime": 45.9794,
+      "eval_samples_per_second": 21.749,
+      "eval_steps_per_second": 0.913,
+      "step": 1620
+    },
+    {
+      "epoch": 2.0174237710018668,
+      "grad_norm": 3.314291000366211,
+      "learning_rate": 1.8542435424354244e-06,
+      "loss": 1.3778,
+      "step": 1621
+    },
+    {
+      "epoch": 2.0186683260734286,
+      "grad_norm": 3.3624770641326904,
+      "learning_rate": 1.8519372693726939e-06,
+      "loss": 1.3948,
+      "step": 1622
+    },
+    {
+      "epoch": 2.0199128811449905,
+      "grad_norm": 3.4041574001312256,
+      "learning_rate": 1.8496309963099633e-06,
+      "loss": 1.3109,
+      "step": 1623
+    },
+    {
+      "epoch": 2.0211574362165527,
+      "grad_norm": 3.5128726959228516,
+      "learning_rate": 1.8473247232472327e-06,
+      "loss": 1.4223,
+      "step": 1624
+    },
+    {
+      "epoch": 2.0224019912881146,
+      "grad_norm": 3.444768190383911,
+      "learning_rate": 1.845018450184502e-06,
+      "loss": 1.3672,
+      "step": 1625
+    },
+    {
+      "epoch": 2.0236465463596764,
+      "grad_norm": 3.478060722351074,
+      "learning_rate": 1.8427121771217713e-06,
+      "loss": 1.4384,
+      "step": 1626
+    },
+    {
+      "epoch": 2.0248911014312383,
+      "grad_norm": 3.211465835571289,
+      "learning_rate": 1.8404059040590408e-06,
+      "loss": 1.3891,
+      "step": 1627
+    },
+    {
+      "epoch": 2.0261356565028,
+      "grad_norm": 3.1283977031707764,
+      "learning_rate": 1.83809963099631e-06,
+      "loss": 1.3752,
+      "step": 1628
+    },
+    {
+      "epoch": 2.027380211574362,
+      "grad_norm": 3.3699934482574463,
+      "learning_rate": 1.8357933579335796e-06,
+      "loss": 1.3751,
+      "step": 1629
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "grad_norm": 3.0729823112487793,
+      "learning_rate": 1.8334870848708488e-06,
+      "loss": 1.3452,
+      "step": 1630
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "eval_loss": 1.6185853481292725,
+      "eval_runtime": 45.2873,
+      "eval_samples_per_second": 22.081,
+      "eval_steps_per_second": 0.927,
+      "step": 1630
+    },
+    {
+      "epoch": 2.029869321717486,
+      "grad_norm": 3.274139642715454,
+      "learning_rate": 1.8311808118081182e-06,
+      "loss": 1.3242,
+      "step": 1631
+    },
+    {
+      "epoch": 2.031113876789048,
+      "grad_norm": 3.2869648933410645,
+      "learning_rate": 1.8288745387453877e-06,
+      "loss": 1.3435,
+      "step": 1632
+    },
+    {
+      "epoch": 2.03235843186061,
+      "grad_norm": 3.3614306449890137,
+      "learning_rate": 1.8265682656826569e-06,
+      "loss": 1.3705,
+      "step": 1633
+    },
+    {
+      "epoch": 2.0336029869321717,
+      "grad_norm": 3.411914587020874,
+      "learning_rate": 1.8242619926199265e-06,
+      "loss": 1.3942,
+      "step": 1634
+    },
+    {
+      "epoch": 2.0348475420037335,
+      "grad_norm": 3.274519920349121,
+      "learning_rate": 1.8219557195571957e-06,
+      "loss": 1.3393,
+      "step": 1635
+    },
+    {
+      "epoch": 2.0360920970752954,
+      "grad_norm": 3.993478775024414,
+      "learning_rate": 1.819649446494465e-06,
+      "loss": 1.3713,
+      "step": 1636
+    },
+    {
+      "epoch": 2.0373366521468577,
+      "grad_norm": 3.273571491241455,
+      "learning_rate": 1.8173431734317346e-06,
+      "loss": 1.3337,
+      "step": 1637
+    },
+    {
+      "epoch": 2.0385812072184195,
+      "grad_norm": 3.1542446613311768,
+      "learning_rate": 1.8150369003690038e-06,
+      "loss": 1.3793,
+      "step": 1638
+    },
+    {
+      "epoch": 2.0398257622899814,
+      "grad_norm": 3.263773202896118,
+      "learning_rate": 1.812730627306273e-06,
+      "loss": 1.3844,
+      "step": 1639
+    },
+    {
+      "epoch": 2.041070317361543,
+      "grad_norm": 3.268772840499878,
+      "learning_rate": 1.8104243542435426e-06,
+      "loss": 1.3947,
+      "step": 1640
+    },
+    {
+      "epoch": 2.041070317361543,
+      "eval_loss": 1.6149734258651733,
+      "eval_runtime": 48.5557,
+      "eval_samples_per_second": 20.595,
+      "eval_steps_per_second": 0.865,
+      "step": 1640
+    },
+    {
+      "epoch": 2.042314872433105,
+      "grad_norm": 3.190168619155884,
+      "learning_rate": 1.8081180811808118e-06,
+      "loss": 1.3276,
+      "step": 1641
+    },
+    {
+      "epoch": 2.043559427504667,
+      "grad_norm": 3.1573212146759033,
+      "learning_rate": 1.8058118081180815e-06,
+      "loss": 1.3307,
+      "step": 1642
+    },
+    {
+      "epoch": 2.044803982576229,
+      "grad_norm": 3.286513328552246,
+      "learning_rate": 1.8035055350553507e-06,
+      "loss": 1.3359,
+      "step": 1643
+    },
+    {
+      "epoch": 2.046048537647791,
+      "grad_norm": 3.342024803161621,
+      "learning_rate": 1.8011992619926199e-06,
+      "loss": 1.3472,
+      "step": 1644
+    },
+    {
+      "epoch": 2.047293092719353,
+      "grad_norm": 3.3237056732177734,
+      "learning_rate": 1.7988929889298895e-06,
+      "loss": 1.3898,
+      "step": 1645
+    },
+    {
+      "epoch": 2.0485376477909147,
+      "grad_norm": 3.370035409927368,
+      "learning_rate": 1.7965867158671587e-06,
+      "loss": 1.3537,
+      "step": 1646
+    },
+    {
+      "epoch": 2.0497822028624766,
+      "grad_norm": 3.1636598110198975,
+      "learning_rate": 1.7942804428044284e-06,
+      "loss": 1.3455,
+      "step": 1647
+    },
+    {
+      "epoch": 2.0510267579340384,
+      "grad_norm": 3.239396810531616,
+      "learning_rate": 1.7919741697416976e-06,
+      "loss": 1.3212,
+      "step": 1648
+    },
+    {
+      "epoch": 2.0522713130056003,
+      "grad_norm": 3.2765328884124756,
+      "learning_rate": 1.7896678966789668e-06,
+      "loss": 1.396,
+      "step": 1649
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "grad_norm": 3.2259459495544434,
+      "learning_rate": 1.7873616236162364e-06,
+      "loss": 1.3614,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "eval_loss": 1.614806056022644,
+      "eval_runtime": 49.8704,
+      "eval_samples_per_second": 20.052,
+      "eval_steps_per_second": 0.842,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0547604231487244,
+      "grad_norm": 3.257683277130127,
+      "learning_rate": 1.7850553505535056e-06,
+      "loss": 1.3663,
+      "step": 1651
+    },
+    {
+      "epoch": 2.0560049782202863,
+      "grad_norm": 3.292070150375366,
+      "learning_rate": 1.7827490774907748e-06,
+      "loss": 1.3548,
+      "step": 1652
+    },
+    {
+      "epoch": 2.057249533291848,
+      "grad_norm": 3.3517003059387207,
+      "learning_rate": 1.7804428044280445e-06,
+      "loss": 1.4015,
+      "step": 1653
+    },
+    {
+      "epoch": 2.05849408836341,
+      "grad_norm": 3.5100908279418945,
+      "learning_rate": 1.7781365313653137e-06,
+      "loss": 1.3666,
+      "step": 1654
+    },
+    {
+      "epoch": 2.059738643434972,
+      "grad_norm": 3.3054633140563965,
+      "learning_rate": 1.7758302583025833e-06,
+      "loss": 1.3691,
+      "step": 1655
+    },
+    {
+      "epoch": 2.060983198506534,
+      "grad_norm": 3.1036577224731445,
+      "learning_rate": 1.7735239852398525e-06,
+      "loss": 1.2721,
+      "step": 1656
+    },
+    {
+      "epoch": 2.062227753578096,
+      "grad_norm": 3.604623317718506,
+      "learning_rate": 1.7712177121771217e-06,
+      "loss": 1.376,
+      "step": 1657
+    },
+    {
+      "epoch": 2.063472308649658,
+      "grad_norm": 3.5246458053588867,
+      "learning_rate": 1.7689114391143914e-06,
+      "loss": 1.3891,
+      "step": 1658
+    },
+    {
+      "epoch": 2.0647168637212197,
+      "grad_norm": 3.286940336227417,
+      "learning_rate": 1.7666051660516606e-06,
+      "loss": 1.3978,
+      "step": 1659
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "grad_norm": 3.1022047996520996,
+      "learning_rate": 1.7642988929889302e-06,
+      "loss": 1.3373,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "eval_loss": 1.6144907474517822,
+      "eval_runtime": 48.4382,
+      "eval_samples_per_second": 20.645,
+      "eval_steps_per_second": 0.867,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0672059738643433,
+      "grad_norm": 3.4111568927764893,
+      "learning_rate": 1.7619926199261994e-06,
+      "loss": 1.3816,
+      "step": 1661
+    },
+    {
+      "epoch": 2.068450528935905,
+      "grad_norm": 3.478947162628174,
+      "learning_rate": 1.7596863468634687e-06,
+      "loss": 1.3601,
+      "step": 1662
+    },
+    {
+      "epoch": 2.0696950840074675,
+      "grad_norm": 3.3320810794830322,
+      "learning_rate": 1.7573800738007383e-06,
+      "loss": 1.3454,
+      "step": 1663
+    },
+    {
+      "epoch": 2.0709396390790293,
+      "grad_norm": 3.475529909133911,
+      "learning_rate": 1.7550738007380075e-06,
+      "loss": 1.3666,
+      "step": 1664
+    },
+    {
+      "epoch": 2.072184194150591,
+      "grad_norm": 3.302342653274536,
+      "learning_rate": 1.752767527675277e-06,
+      "loss": 1.3824,
+      "step": 1665
+    },
+    {
+      "epoch": 2.073428749222153,
+      "grad_norm": 3.198641777038574,
+      "learning_rate": 1.7504612546125463e-06,
+      "loss": 1.3398,
+      "step": 1666
+    },
+    {
+      "epoch": 2.074673304293715,
+      "grad_norm": 3.421834945678711,
+      "learning_rate": 1.7481549815498156e-06,
+      "loss": 1.363,
+      "step": 1667
+    },
+    {
+      "epoch": 2.0759178593652767,
+      "grad_norm": 3.654343843460083,
+      "learning_rate": 1.7458487084870852e-06,
+      "loss": 1.3852,
+      "step": 1668
+    },
+    {
+      "epoch": 2.077162414436839,
+      "grad_norm": 3.3543035984039307,
+      "learning_rate": 1.7435424354243544e-06,
+      "loss": 1.3554,
+      "step": 1669
+    },
+    {
+      "epoch": 2.078406969508401,
+      "grad_norm": 3.2144811153411865,
+      "learning_rate": 1.7412361623616236e-06,
+      "loss": 1.3137,
+      "step": 1670
+    },
+    {
+      "epoch": 2.078406969508401,
+      "eval_loss": 1.6155465841293335,
+      "eval_runtime": 52.4111,
+      "eval_samples_per_second": 19.08,
+      "eval_steps_per_second": 0.801,
+      "step": 1670
+    },
+    {
+      "epoch": 2.0796515245799627,
+      "grad_norm": 3.2240991592407227,
+      "learning_rate": 1.7389298892988932e-06,
+      "loss": 1.3578,
+      "step": 1671
+    },
+    {
+      "epoch": 2.0808960796515246,
+      "grad_norm": 3.1450018882751465,
+      "learning_rate": 1.7366236162361625e-06,
+      "loss": 1.2999,
+      "step": 1672
+    },
+    {
+      "epoch": 2.0821406347230864,
+      "grad_norm": 3.4396843910217285,
+      "learning_rate": 1.7343173431734319e-06,
+      "loss": 1.3522,
+      "step": 1673
+    },
+    {
+      "epoch": 2.0833851897946483,
+      "grad_norm": 3.1719677448272705,
+      "learning_rate": 1.7320110701107013e-06,
+      "loss": 1.3552,
+      "step": 1674
+    },
+    {
+      "epoch": 2.08462974486621,
+      "grad_norm": 3.4420199394226074,
+      "learning_rate": 1.7297047970479705e-06,
+      "loss": 1.4033,
+      "step": 1675
+    },
+    {
+      "epoch": 2.0858742999377724,
+      "grad_norm": 3.3366570472717285,
+      "learning_rate": 1.7273985239852401e-06,
+      "loss": 1.3407,
+      "step": 1676
+    },
+    {
+      "epoch": 2.0871188550093343,
+      "grad_norm": 3.3416335582733154,
+      "learning_rate": 1.7250922509225094e-06,
+      "loss": 1.3882,
+      "step": 1677
+    },
+    {
+      "epoch": 2.088363410080896,
+      "grad_norm": 3.4189534187316895,
+      "learning_rate": 1.7227859778597788e-06,
+      "loss": 1.3996,
+      "step": 1678
+    },
+    {
+      "epoch": 2.089607965152458,
+      "grad_norm": 3.309095859527588,
+      "learning_rate": 1.7204797047970482e-06,
+      "loss": 1.3799,
+      "step": 1679
+    },
+    {
+      "epoch": 2.09085252022402,
+      "grad_norm": 3.369511365890503,
+      "learning_rate": 1.7181734317343174e-06,
+      "loss": 1.4159,
+      "step": 1680
+    },
+    {
+      "epoch": 2.09085252022402,
+      "eval_loss": 1.6145448684692383,
+      "eval_runtime": 49.0228,
+      "eval_samples_per_second": 20.399,
+      "eval_steps_per_second": 0.857,
+      "step": 1680
+    },
+    {
+      "epoch": 2.0920970752955816,
+      "grad_norm": 3.323286771774292,
+      "learning_rate": 1.7158671586715868e-06,
+      "loss": 1.4035,
+      "step": 1681
+    },
+    {
+      "epoch": 2.093341630367144,
+      "grad_norm": 3.3053157329559326,
+      "learning_rate": 1.7135608856088563e-06,
+      "loss": 1.3498,
+      "step": 1682
+    },
+    {
+      "epoch": 2.094586185438706,
+      "grad_norm": 3.141890048980713,
+      "learning_rate": 1.7112546125461257e-06,
+      "loss": 1.3438,
+      "step": 1683
+    },
+    {
+      "epoch": 2.0958307405102676,
+      "grad_norm": 3.135058879852295,
+      "learning_rate": 1.708948339483395e-06,
+      "loss": 1.3726,
+      "step": 1684
+    },
+    {
+      "epoch": 2.0970752955818295,
+      "grad_norm": 3.2389063835144043,
+      "learning_rate": 1.7066420664206643e-06,
+      "loss": 1.3661,
+      "step": 1685
+    },
+    {
+      "epoch": 2.0983198506533913,
+      "grad_norm": 3.1741721630096436,
+      "learning_rate": 1.7043357933579337e-06,
+      "loss": 1.3438,
+      "step": 1686
+    },
+    {
+      "epoch": 2.099564405724953,
+      "grad_norm": 3.3524587154388428,
+      "learning_rate": 1.7020295202952032e-06,
+      "loss": 1.3388,
+      "step": 1687
+    },
+    {
+      "epoch": 2.100808960796515,
+      "grad_norm": 3.3574740886688232,
+      "learning_rate": 1.6997232472324724e-06,
+      "loss": 1.3702,
+      "step": 1688
+    },
+    {
+      "epoch": 2.1020535158680773,
+      "grad_norm": 3.458645820617676,
+      "learning_rate": 1.6974169741697418e-06,
+      "loss": 1.3785,
+      "step": 1689
+    },
+    {
+      "epoch": 2.103298070939639,
+      "grad_norm": 3.346677303314209,
+      "learning_rate": 1.6951107011070112e-06,
+      "loss": 1.3295,
+      "step": 1690
+    },
+    {
+      "epoch": 2.103298070939639,
+      "eval_loss": 1.6184555292129517,
+      "eval_runtime": 46.6544,
+      "eval_samples_per_second": 21.434,
+      "eval_steps_per_second": 0.9,
+      "step": 1690
+    },
+    {
+      "epoch": 2.104542626011201,
+      "grad_norm": 3.3403804302215576,
+      "learning_rate": 1.6928044280442806e-06,
+      "loss": 1.3912,
+      "step": 1691
+    },
+    {
+      "epoch": 2.105787181082763,
+      "grad_norm": 3.2638344764709473,
+      "learning_rate": 1.6904981549815499e-06,
+      "loss": 1.3684,
+      "step": 1692
+    },
+    {
+      "epoch": 2.1070317361543247,
+      "grad_norm": 3.192701578140259,
+      "learning_rate": 1.6881918819188193e-06,
+      "loss": 1.2997,
+      "step": 1693
+    },
+    {
+      "epoch": 2.1082762912258866,
+      "grad_norm": 3.482121706008911,
+      "learning_rate": 1.6858856088560887e-06,
+      "loss": 1.3515,
+      "step": 1694
+    },
+    {
+      "epoch": 2.109520846297449,
+      "grad_norm": 3.292353630065918,
+      "learning_rate": 1.6835793357933581e-06,
+      "loss": 1.3603,
+      "step": 1695
+    },
+    {
+      "epoch": 2.1107654013690107,
+      "grad_norm": 3.34220290184021,
+      "learning_rate": 1.6812730627306275e-06,
+      "loss": 1.3666,
+      "step": 1696
+    },
+    {
+      "epoch": 2.1120099564405725,
+      "grad_norm": 3.190058469772339,
+      "learning_rate": 1.6789667896678968e-06,
+      "loss": 1.3549,
+      "step": 1697
+    },
+    {
+      "epoch": 2.1132545115121344,
+      "grad_norm": 3.3759775161743164,
+      "learning_rate": 1.6766605166051662e-06,
+      "loss": 1.3628,
+      "step": 1698
+    },
+    {
+      "epoch": 2.1144990665836962,
+      "grad_norm": 3.419440507888794,
+      "learning_rate": 1.6743542435424356e-06,
+      "loss": 1.3618,
+      "step": 1699
+    },
+    {
+      "epoch": 2.115743621655258,
+      "grad_norm": 3.5021324157714844,
+      "learning_rate": 1.6720479704797048e-06,
+      "loss": 1.4023,
+      "step": 1700
+    },
+    {
+      "epoch": 2.115743621655258,
+      "eval_loss": 1.6218053102493286,
+      "eval_runtime": 48.3483,
+      "eval_samples_per_second": 20.683,
+      "eval_steps_per_second": 0.869,
+      "step": 1700
+    },
+    {
+      "epoch": 2.11698817672682,
+      "grad_norm": 3.449636936187744,
+      "learning_rate": 1.6697416974169742e-06,
+      "loss": 1.3534,
+      "step": 1701
+    },
+    {
+      "epoch": 2.1182327317983822,
+      "grad_norm": 3.483173370361328,
+      "learning_rate": 1.6674354243542437e-06,
+      "loss": 1.4053,
+      "step": 1702
+    },
+    {
+      "epoch": 2.119477286869944,
+      "grad_norm": 3.3713502883911133,
+      "learning_rate": 1.665129151291513e-06,
+      "loss": 1.3352,
+      "step": 1703
+    },
+    {
+      "epoch": 2.120721841941506,
+      "grad_norm": 3.2565910816192627,
+      "learning_rate": 1.6628228782287825e-06,
+      "loss": 1.3751,
+      "step": 1704
+    },
+    {
+      "epoch": 2.1219663970130678,
+      "grad_norm": 3.1434268951416016,
+      "learning_rate": 1.6605166051660517e-06,
+      "loss": 1.346,
+      "step": 1705
+    },
+    {
+      "epoch": 2.1232109520846296,
+      "grad_norm": 3.229909896850586,
+      "learning_rate": 1.6582103321033211e-06,
+      "loss": 1.3477,
+      "step": 1706
+    },
+    {
+      "epoch": 2.1244555071561915,
+      "grad_norm": 3.3930044174194336,
+      "learning_rate": 1.6559040590405906e-06,
+      "loss": 1.3374,
+      "step": 1707
+    },
+    {
+      "epoch": 2.1257000622277538,
+      "grad_norm": 3.2674202919006348,
+      "learning_rate": 1.6535977859778598e-06,
+      "loss": 1.321,
+      "step": 1708
+    },
+    {
+      "epoch": 2.1269446172993156,
+      "grad_norm": 3.3048641681671143,
+      "learning_rate": 1.6512915129151294e-06,
+      "loss": 1.3641,
+      "step": 1709
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "grad_norm": 3.4714198112487793,
+      "learning_rate": 1.6489852398523986e-06,
+      "loss": 1.3931,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "eval_loss": 1.6198205947875977,
+      "eval_runtime": 42.9272,
+      "eval_samples_per_second": 23.295,
+      "eval_steps_per_second": 0.978,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1294337274424393,
+      "grad_norm": 3.440058946609497,
+      "learning_rate": 1.6466789667896678e-06,
+      "loss": 1.3892,
+      "step": 1711
+    },
+    {
+      "epoch": 2.130678282514001,
+      "grad_norm": 3.4067635536193848,
+      "learning_rate": 1.6443726937269375e-06,
+      "loss": 1.3719,
+      "step": 1712
+    },
+    {
+      "epoch": 2.131922837585563,
+      "grad_norm": 3.285536766052246,
+      "learning_rate": 1.6420664206642067e-06,
+      "loss": 1.3735,
+      "step": 1713
+    },
+    {
+      "epoch": 2.133167392657125,
+      "grad_norm": 3.171344041824341,
+      "learning_rate": 1.6397601476014763e-06,
+      "loss": 1.3628,
+      "step": 1714
+    },
+    {
+      "epoch": 2.134411947728687,
+      "grad_norm": 3.3051321506500244,
+      "learning_rate": 1.6374538745387455e-06,
+      "loss": 1.3579,
+      "step": 1715
+    },
+    {
+      "epoch": 2.135656502800249,
+      "grad_norm": 3.2083475589752197,
+      "learning_rate": 1.6351476014760147e-06,
+      "loss": 1.297,
+      "step": 1716
+    },
+    {
+      "epoch": 2.136901057871811,
+      "grad_norm": 3.340172052383423,
+      "learning_rate": 1.6328413284132844e-06,
+      "loss": 1.3468,
+      "step": 1717
+    },
+    {
+      "epoch": 2.1381456129433727,
+      "grad_norm": 3.3475770950317383,
+      "learning_rate": 1.6305350553505536e-06,
+      "loss": 1.3918,
+      "step": 1718
+    },
+    {
+      "epoch": 2.1393901680149345,
+      "grad_norm": 3.4087703227996826,
+      "learning_rate": 1.6282287822878228e-06,
+      "loss": 1.4083,
+      "step": 1719
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "grad_norm": 3.461160898208618,
+      "learning_rate": 1.6259225092250924e-06,
+      "loss": 1.3471,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "eval_loss": 1.6216135025024414,
+      "eval_runtime": 49.1097,
+      "eval_samples_per_second": 20.363,
+      "eval_steps_per_second": 0.855,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1418792781580587,
+      "grad_norm": 3.3619799613952637,
+      "learning_rate": 1.6236162361623616e-06,
+      "loss": 1.3235,
+      "step": 1721
+    },
+    {
+      "epoch": 2.1431238332296205,
+      "grad_norm": 3.332010507583618,
+      "learning_rate": 1.6213099630996313e-06,
+      "loss": 1.3576,
+      "step": 1722
+    },
+    {
+      "epoch": 2.1443683883011824,
+      "grad_norm": 3.300055503845215,
+      "learning_rate": 1.6190036900369005e-06,
+      "loss": 1.3269,
+      "step": 1723
+    },
+    {
+      "epoch": 2.145612943372744,
+      "grad_norm": 3.3326926231384277,
+      "learning_rate": 1.6166974169741697e-06,
+      "loss": 1.346,
+      "step": 1724
+    },
+    {
+      "epoch": 2.146857498444306,
+      "grad_norm": 3.4713706970214844,
+      "learning_rate": 1.6143911439114393e-06,
+      "loss": 1.3416,
+      "step": 1725
+    },
+    {
+      "epoch": 2.148102053515868,
+      "grad_norm": 3.1899967193603516,
+      "learning_rate": 1.6120848708487085e-06,
+      "loss": 1.3192,
+      "step": 1726
+    },
+    {
+      "epoch": 2.1493466085874298,
+      "grad_norm": 3.1781857013702393,
+      "learning_rate": 1.6097785977859782e-06,
+      "loss": 1.3197,
+      "step": 1727
+    },
+    {
+      "epoch": 2.150591163658992,
+      "grad_norm": 3.3369524478912354,
+      "learning_rate": 1.6074723247232474e-06,
+      "loss": 1.3804,
+      "step": 1728
+    },
+    {
+      "epoch": 2.151835718730554,
+      "grad_norm": 3.2813405990600586,
+      "learning_rate": 1.6051660516605166e-06,
+      "loss": 1.3778,
+      "step": 1729
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "grad_norm": 3.3395135402679443,
+      "learning_rate": 1.6028597785977862e-06,
+      "loss": 1.3523,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "eval_loss": 1.6181366443634033,
+      "eval_runtime": 50.7133,
+      "eval_samples_per_second": 19.719,
+      "eval_steps_per_second": 0.828,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1543248288736776,
+      "grad_norm": 3.2905092239379883,
+      "learning_rate": 1.6005535055350554e-06,
+      "loss": 1.3568,
+      "step": 1731
+    },
+    {
+      "epoch": 2.1555693839452394,
+      "grad_norm": 3.314086437225342,
+      "learning_rate": 1.5982472324723247e-06,
+      "loss": 1.3543,
+      "step": 1732
+    },
+    {
+      "epoch": 2.1568139390168013,
+      "grad_norm": 3.340034008026123,
+      "learning_rate": 1.5959409594095943e-06,
+      "loss": 1.3639,
+      "step": 1733
+    },
+    {
+      "epoch": 2.1580584940883636,
+      "grad_norm": 3.264814615249634,
+      "learning_rate": 1.5936346863468635e-06,
+      "loss": 1.3599,
+      "step": 1734
+    },
+    {
+      "epoch": 2.1593030491599254,
+      "grad_norm": 3.2429778575897217,
+      "learning_rate": 1.5913284132841331e-06,
+      "loss": 1.3379,
+      "step": 1735
+    },
+    {
+      "epoch": 2.1605476042314873,
+      "grad_norm": 3.221935987472534,
+      "learning_rate": 1.5890221402214023e-06,
+      "loss": 1.3162,
+      "step": 1736
+    },
+    {
+      "epoch": 2.161792159303049,
+      "grad_norm": 3.266073703765869,
+      "learning_rate": 1.5867158671586716e-06,
+      "loss": 1.3632,
+      "step": 1737
+    },
+    {
+      "epoch": 2.163036714374611,
+      "grad_norm": 3.2618112564086914,
+      "learning_rate": 1.5844095940959412e-06,
+      "loss": 1.3779,
+      "step": 1738
+    },
+    {
+      "epoch": 2.164281269446173,
+      "grad_norm": 3.2474825382232666,
+      "learning_rate": 1.5821033210332104e-06,
+      "loss": 1.3201,
+      "step": 1739
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "grad_norm": 3.4494247436523438,
+      "learning_rate": 1.57979704797048e-06,
+      "loss": 1.3949,
+      "step": 1740
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "eval_loss": 1.6180604696273804,
+      "eval_runtime": 49.3152,
+      "eval_samples_per_second": 20.278,
+      "eval_steps_per_second": 0.852,
+      "step": 1740
+    },
+    {
+      "epoch": 2.166770379589297,
+      "grad_norm": 3.221304416656494,
+      "learning_rate": 1.5774907749077492e-06,
+      "loss": 1.3341,
+      "step": 1741
+    },
+    {
+      "epoch": 2.168014934660859,
+      "grad_norm": 3.28930401802063,
+      "learning_rate": 1.5751845018450185e-06,
+      "loss": 1.3151,
+      "step": 1742
+    },
+    {
+      "epoch": 2.1692594897324207,
+      "grad_norm": 3.3582026958465576,
+      "learning_rate": 1.572878228782288e-06,
+      "loss": 1.3767,
+      "step": 1743
+    },
+    {
+      "epoch": 2.1705040448039825,
+      "grad_norm": 3.2603683471679688,
+      "learning_rate": 1.5705719557195573e-06,
+      "loss": 1.3644,
+      "step": 1744
+    },
+    {
+      "epoch": 2.1717485998755444,
+      "grad_norm": 3.3291070461273193,
+      "learning_rate": 1.5682656826568267e-06,
+      "loss": 1.3226,
+      "step": 1745
+    },
+    {
+      "epoch": 2.172993154947106,
+      "grad_norm": 3.473127841949463,
+      "learning_rate": 1.5659594095940962e-06,
+      "loss": 1.3531,
+      "step": 1746
+    },
+    {
+      "epoch": 2.1742377100186685,
+      "grad_norm": 3.2014408111572266,
+      "learning_rate": 1.5636531365313654e-06,
+      "loss": 1.3634,
+      "step": 1747
+    },
+    {
+      "epoch": 2.1754822650902303,
+      "grad_norm": 3.56447434425354,
+      "learning_rate": 1.561346863468635e-06,
+      "loss": 1.3788,
+      "step": 1748
+    },
+    {
+      "epoch": 2.176726820161792,
+      "grad_norm": 3.351318359375,
+      "learning_rate": 1.5590405904059042e-06,
+      "loss": 1.3133,
+      "step": 1749
+    },
+    {
+      "epoch": 2.177971375233354,
+      "grad_norm": 3.278183937072754,
+      "learning_rate": 1.5567343173431734e-06,
+      "loss": 1.3164,
+      "step": 1750
+    },
+    {
+      "epoch": 2.177971375233354,
+      "eval_loss": 1.6197904348373413,
+      "eval_runtime": 50.2776,
+      "eval_samples_per_second": 19.89,
+      "eval_steps_per_second": 0.835,
+      "step": 1750
+    },
+    {
+      "epoch": 2.179215930304916,
+      "grad_norm": 3.379854917526245,
+      "learning_rate": 1.554428044280443e-06,
+      "loss": 1.3785,
+      "step": 1751
+    },
+    {
+      "epoch": 2.1804604853764777,
+      "grad_norm": 3.2265610694885254,
+      "learning_rate": 1.5521217712177123e-06,
+      "loss": 1.3483,
+      "step": 1752
+    },
+    {
+      "epoch": 2.18170504044804,
+      "grad_norm": 3.2259609699249268,
+      "learning_rate": 1.5498154981549817e-06,
+      "loss": 1.377,
+      "step": 1753
+    },
+    {
+      "epoch": 2.182949595519602,
+      "grad_norm": 3.1671273708343506,
+      "learning_rate": 1.5475092250922511e-06,
+      "loss": 1.3106,
+      "step": 1754
+    },
+    {
+      "epoch": 2.1841941505911637,
+      "grad_norm": 3.280299425125122,
+      "learning_rate": 1.5452029520295203e-06,
+      "loss": 1.3271,
+      "step": 1755
+    },
+    {
+      "epoch": 2.1854387056627256,
+      "grad_norm": 3.224238634109497,
+      "learning_rate": 1.5428966789667897e-06,
+      "loss": 1.3369,
+      "step": 1756
+    },
+    {
+      "epoch": 2.1866832607342874,
+      "grad_norm": 3.3139021396636963,
+      "learning_rate": 1.5405904059040592e-06,
+      "loss": 1.3562,
+      "step": 1757
+    },
+    {
+      "epoch": 2.1879278158058493,
+      "grad_norm": 3.340569257736206,
+      "learning_rate": 1.5382841328413286e-06,
+      "loss": 1.3526,
+      "step": 1758
+    },
+    {
+      "epoch": 2.189172370877411,
+      "grad_norm": 3.505159854888916,
+      "learning_rate": 1.535977859778598e-06,
+      "loss": 1.3796,
+      "step": 1759
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "grad_norm": 3.351260185241699,
+      "learning_rate": 1.5336715867158672e-06,
+      "loss": 1.3676,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "eval_loss": 1.6169729232788086,
+      "eval_runtime": 48.0801,
+      "eval_samples_per_second": 20.799,
+      "eval_steps_per_second": 0.874,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1916614810205353,
+      "grad_norm": 3.4291462898254395,
+      "learning_rate": 1.5313653136531366e-06,
+      "loss": 1.3574,
+      "step": 1761
+    },
+    {
+      "epoch": 2.192906036092097,
+      "grad_norm": 3.5459227561950684,
+      "learning_rate": 1.529059040590406e-06,
+      "loss": 1.3919,
+      "step": 1762
+    },
+    {
+      "epoch": 2.194150591163659,
+      "grad_norm": 3.298430919647217,
+      "learning_rate": 1.5267527675276755e-06,
+      "loss": 1.3551,
+      "step": 1763
+    },
+    {
+      "epoch": 2.195395146235221,
+      "grad_norm": 3.4521286487579346,
+      "learning_rate": 1.5244464944649447e-06,
+      "loss": 1.3779,
+      "step": 1764
+    },
+    {
+      "epoch": 2.1966397013067827,
+      "grad_norm": 3.354443073272705,
+      "learning_rate": 1.5221402214022141e-06,
+      "loss": 1.3574,
+      "step": 1765
+    },
+    {
+      "epoch": 2.197884256378345,
+      "grad_norm": 3.069854259490967,
+      "learning_rate": 1.5198339483394836e-06,
+      "loss": 1.2987,
+      "step": 1766
+    },
+    {
+      "epoch": 2.199128811449907,
+      "grad_norm": 3.0894274711608887,
+      "learning_rate": 1.517527675276753e-06,
+      "loss": 1.3408,
+      "step": 1767
+    },
+    {
+      "epoch": 2.2003733665214686,
+      "grad_norm": 3.3033676147460938,
+      "learning_rate": 1.5152214022140222e-06,
+      "loss": 1.3394,
+      "step": 1768
+    },
+    {
+      "epoch": 2.2016179215930305,
+      "grad_norm": 3.2894108295440674,
+      "learning_rate": 1.5129151291512916e-06,
+      "loss": 1.3505,
+      "step": 1769
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "grad_norm": 3.263564109802246,
+      "learning_rate": 1.510608856088561e-06,
+      "loss": 1.3318,
+      "step": 1770
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "eval_loss": 1.6183674335479736,
+      "eval_runtime": 48.3189,
+      "eval_samples_per_second": 20.696,
+      "eval_steps_per_second": 0.869,
+      "step": 1770
+    },
+    {
+      "epoch": 2.204107031736154,
+      "grad_norm": 3.2933449745178223,
+      "learning_rate": 1.5083025830258305e-06,
+      "loss": 1.3397,
+      "step": 1771
+    },
+    {
+      "epoch": 2.205351586807716,
+      "grad_norm": 3.3185815811157227,
+      "learning_rate": 1.5059963099630997e-06,
+      "loss": 1.3339,
+      "step": 1772
+    },
+    {
+      "epoch": 2.2065961418792783,
+      "grad_norm": 3.445251703262329,
+      "learning_rate": 1.503690036900369e-06,
+      "loss": 1.3898,
+      "step": 1773
+    },
+    {
+      "epoch": 2.20784069695084,
+      "grad_norm": 3.399627685546875,
+      "learning_rate": 1.5013837638376385e-06,
+      "loss": 1.3646,
+      "step": 1774
+    },
+    {
+      "epoch": 2.209085252022402,
+      "grad_norm": 3.3483574390411377,
+      "learning_rate": 1.499077490774908e-06,
+      "loss": 1.3732,
+      "step": 1775
+    },
+    {
+      "epoch": 2.210329807093964,
+      "grad_norm": 3.3097689151763916,
+      "learning_rate": 1.4967712177121774e-06,
+      "loss": 1.3317,
+      "step": 1776
+    },
+    {
+      "epoch": 2.2115743621655257,
+      "grad_norm": 3.254025459289551,
+      "learning_rate": 1.4944649446494466e-06,
+      "loss": 1.3818,
+      "step": 1777
+    },
+    {
+      "epoch": 2.2128189172370876,
+      "grad_norm": 3.2063064575195312,
+      "learning_rate": 1.492158671586716e-06,
+      "loss": 1.3357,
+      "step": 1778
+    },
+    {
+      "epoch": 2.21406347230865,
+      "grad_norm": 3.1974005699157715,
+      "learning_rate": 1.4898523985239854e-06,
+      "loss": 1.4103,
+      "step": 1779
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "grad_norm": 3.1551153659820557,
+      "learning_rate": 1.4875461254612546e-06,
+      "loss": 1.3088,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "eval_loss": 1.6195781230926514,
+      "eval_runtime": 52.379,
+      "eval_samples_per_second": 19.092,
+      "eval_steps_per_second": 0.802,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2165525824517736,
+      "grad_norm": 3.3186404705047607,
+      "learning_rate": 1.485239852398524e-06,
+      "loss": 1.3775,
+      "step": 1781
+    },
+    {
+      "epoch": 2.2177971375233354,
+      "grad_norm": 3.2764358520507812,
+      "learning_rate": 1.4829335793357935e-06,
+      "loss": 1.3253,
+      "step": 1782
+    },
+    {
+      "epoch": 2.2190416925948973,
+      "grad_norm": 3.3535802364349365,
+      "learning_rate": 1.4806273062730627e-06,
+      "loss": 1.3451,
+      "step": 1783
+    },
+    {
+      "epoch": 2.220286247666459,
+      "grad_norm": 3.304255962371826,
+      "learning_rate": 1.4783210332103323e-06,
+      "loss": 1.3565,
+      "step": 1784
+    },
+    {
+      "epoch": 2.2215308027380214,
+      "grad_norm": 3.281244993209839,
+      "learning_rate": 1.4760147601476015e-06,
+      "loss": 1.3458,
+      "step": 1785
+    },
+    {
+      "epoch": 2.2227753578095832,
+      "grad_norm": 3.4199814796447754,
+      "learning_rate": 1.473708487084871e-06,
+      "loss": 1.359,
+      "step": 1786
+    },
+    {
+      "epoch": 2.224019912881145,
+      "grad_norm": 3.298577070236206,
+      "learning_rate": 1.4714022140221404e-06,
+      "loss": 1.3298,
+      "step": 1787
+    },
+    {
+      "epoch": 2.225264467952707,
+      "grad_norm": 3.2206099033355713,
+      "learning_rate": 1.4690959409594096e-06,
+      "loss": 1.3026,
+      "step": 1788
+    },
+    {
+      "epoch": 2.226509023024269,
+      "grad_norm": 3.3583920001983643,
+      "learning_rate": 1.4667896678966792e-06,
+      "loss": 1.3618,
+      "step": 1789
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "grad_norm": 3.4446728229522705,
+      "learning_rate": 1.4644833948339484e-06,
+      "loss": 1.3895,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "eval_loss": 1.6163197755813599,
+      "eval_runtime": 49.4171,
+      "eval_samples_per_second": 20.236,
+      "eval_steps_per_second": 0.85,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2289981331673925,
+      "grad_norm": 3.329155445098877,
+      "learning_rate": 1.4621771217712176e-06,
+      "loss": 1.3368,
+      "step": 1791
+    },
+    {
+      "epoch": 2.2302426882389548,
+      "grad_norm": 3.4599087238311768,
+      "learning_rate": 1.4598708487084873e-06,
+      "loss": 1.3487,
+      "step": 1792
+    },
+    {
+      "epoch": 2.2314872433105166,
+      "grad_norm": 3.3162965774536133,
+      "learning_rate": 1.4575645756457565e-06,
+      "loss": 1.3493,
+      "step": 1793
+    },
+    {
+      "epoch": 2.2327317983820785,
+      "grad_norm": 3.1206021308898926,
+      "learning_rate": 1.4552583025830261e-06,
+      "loss": 1.3506,
+      "step": 1794
+    },
+    {
+      "epoch": 2.2339763534536403,
+      "grad_norm": 3.334165573120117,
+      "learning_rate": 1.4529520295202953e-06,
+      "loss": 1.3315,
+      "step": 1795
+    },
+    {
+      "epoch": 2.235220908525202,
+      "grad_norm": 3.3404204845428467,
+      "learning_rate": 1.4506457564575645e-06,
+      "loss": 1.3163,
+      "step": 1796
+    },
+    {
+      "epoch": 2.236465463596764,
+      "grad_norm": 3.2753002643585205,
+      "learning_rate": 1.4483394833948342e-06,
+      "loss": 1.3567,
+      "step": 1797
+    },
+    {
+      "epoch": 2.2377100186683263,
+      "grad_norm": 3.42242431640625,
+      "learning_rate": 1.4460332103321034e-06,
+      "loss": 1.3761,
+      "step": 1798
+    },
+    {
+      "epoch": 2.238954573739888,
+      "grad_norm": 3.31380033493042,
+      "learning_rate": 1.4437269372693726e-06,
+      "loss": 1.3353,
+      "step": 1799
+    },
+    {
+      "epoch": 2.24019912881145,
+      "grad_norm": 3.239811658859253,
+      "learning_rate": 1.4414206642066422e-06,
+      "loss": 1.3187,
+      "step": 1800
+    },
+    {
+      "epoch": 2.24019912881145,
+      "eval_loss": 1.6201411485671997,
+      "eval_runtime": 53.5369,
+      "eval_samples_per_second": 18.679,
+      "eval_steps_per_second": 0.785,
+      "step": 1800
+    },
+    {
+      "epoch": 2.241443683883012,
+      "grad_norm": 3.549010753631592,
+      "learning_rate": 1.4391143911439114e-06,
+      "loss": 1.3633,
+      "step": 1801
+    },
+    {
+      "epoch": 2.2426882389545737,
+      "grad_norm": 3.1888253688812256,
+      "learning_rate": 1.436808118081181e-06,
+      "loss": 1.3292,
+      "step": 1802
+    },
+    {
+      "epoch": 2.2439327940261355,
+      "grad_norm": 3.1015641689300537,
+      "learning_rate": 1.4345018450184503e-06,
+      "loss": 1.3404,
+      "step": 1803
+    },
+    {
+      "epoch": 2.2451773490976974,
+      "grad_norm": 3.2635490894317627,
+      "learning_rate": 1.4321955719557195e-06,
+      "loss": 1.3181,
+      "step": 1804
+    },
+    {
+      "epoch": 2.2464219041692597,
+      "grad_norm": 3.314697265625,
+      "learning_rate": 1.4298892988929891e-06,
+      "loss": 1.308,
+      "step": 1805
+    },
+    {
+      "epoch": 2.2476664592408215,
+      "grad_norm": 3.3572986125946045,
+      "learning_rate": 1.4275830258302584e-06,
+      "loss": 1.2888,
+      "step": 1806
+    },
+    {
+      "epoch": 2.2489110143123834,
+      "grad_norm": 3.403423547744751,
+      "learning_rate": 1.425276752767528e-06,
+      "loss": 1.3438,
+      "step": 1807
+    },
+    {
+      "epoch": 2.2501555693839452,
+      "grad_norm": 3.5180821418762207,
+      "learning_rate": 1.4229704797047972e-06,
+      "loss": 1.3613,
+      "step": 1808
+    },
+    {
+      "epoch": 2.251400124455507,
+      "grad_norm": 3.5315089225769043,
+      "learning_rate": 1.4206642066420664e-06,
+      "loss": 1.3655,
+      "step": 1809
+    },
+    {
+      "epoch": 2.252644679527069,
+      "grad_norm": 3.4266903400421143,
+      "learning_rate": 1.418357933579336e-06,
+      "loss": 1.3481,
+      "step": 1810
+    },
+    {
+      "epoch": 2.252644679527069,
+      "eval_loss": 1.619051456451416,
+      "eval_runtime": 43.1595,
+      "eval_samples_per_second": 23.17,
+      "eval_steps_per_second": 0.973,
+      "step": 1810
+    },
+    {
+      "epoch": 2.253889234598631,
+      "grad_norm": 3.412053108215332,
+      "learning_rate": 1.4160516605166053e-06,
+      "loss": 1.3097,
+      "step": 1811
+    },
+    {
+      "epoch": 2.255133789670193,
+      "grad_norm": 3.4371933937072754,
+      "learning_rate": 1.4137453874538745e-06,
+      "loss": 1.3831,
+      "step": 1812
+    },
+    {
+      "epoch": 2.256378344741755,
+      "grad_norm": 3.331648588180542,
+      "learning_rate": 1.411439114391144e-06,
+      "loss": 1.3701,
+      "step": 1813
+    },
+    {
+      "epoch": 2.2576228998133168,
+      "grad_norm": 3.3840672969818115,
+      "learning_rate": 1.4091328413284133e-06,
+      "loss": 1.3676,
+      "step": 1814
+    },
+    {
+      "epoch": 2.2588674548848786,
+      "grad_norm": 3.3289988040924072,
+      "learning_rate": 1.406826568265683e-06,
+      "loss": 1.3281,
+      "step": 1815
+    },
+    {
+      "epoch": 2.2601120099564405,
+      "grad_norm": 3.309511423110962,
+      "learning_rate": 1.4045202952029522e-06,
+      "loss": 1.3216,
+      "step": 1816
+    },
+    {
+      "epoch": 2.2613565650280023,
+      "grad_norm": 3.3434019088745117,
+      "learning_rate": 1.4022140221402214e-06,
+      "loss": 1.3261,
+      "step": 1817
+    },
+    {
+      "epoch": 2.2626011200995646,
+      "grad_norm": 3.3371620178222656,
+      "learning_rate": 1.399907749077491e-06,
+      "loss": 1.3597,
+      "step": 1818
+    },
+    {
+      "epoch": 2.2638456751711264,
+      "grad_norm": 3.287353515625,
+      "learning_rate": 1.3976014760147602e-06,
+      "loss": 1.3431,
+      "step": 1819
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "grad_norm": 3.422102689743042,
+      "learning_rate": 1.3952952029520298e-06,
+      "loss": 1.3454,
+      "step": 1820
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "eval_loss": 1.6200023889541626,
+      "eval_runtime": 46.3983,
+      "eval_samples_per_second": 21.552,
+      "eval_steps_per_second": 0.905,
+      "step": 1820
+    },
+    {
+      "epoch": 2.26633478531425,
+      "grad_norm": 3.6252498626708984,
+      "learning_rate": 1.392988929889299e-06,
+      "loss": 1.4066,
+      "step": 1821
+    },
+    {
+      "epoch": 2.267579340385812,
+      "grad_norm": 3.423708915710449,
+      "learning_rate": 1.3906826568265683e-06,
+      "loss": 1.3504,
+      "step": 1822
+    },
+    {
+      "epoch": 2.268823895457374,
+      "grad_norm": 3.3063361644744873,
+      "learning_rate": 1.388376383763838e-06,
+      "loss": 1.3635,
+      "step": 1823
+    },
+    {
+      "epoch": 2.270068450528936,
+      "grad_norm": 3.380657911300659,
+      "learning_rate": 1.3860701107011071e-06,
+      "loss": 1.3442,
+      "step": 1824
+    },
+    {
+      "epoch": 2.271313005600498,
+      "grad_norm": 3.300102472305298,
+      "learning_rate": 1.3837638376383765e-06,
+      "loss": 1.3741,
+      "step": 1825
+    },
+    {
+      "epoch": 2.27255756067206,
+      "grad_norm": 3.3895652294158936,
+      "learning_rate": 1.381457564575646e-06,
+      "loss": 1.3288,
+      "step": 1826
+    },
+    {
+      "epoch": 2.2738021157436217,
+      "grad_norm": 3.4393489360809326,
+      "learning_rate": 1.3791512915129152e-06,
+      "loss": 1.3544,
+      "step": 1827
+    },
+    {
+      "epoch": 2.2750466708151835,
+      "grad_norm": 3.4510536193847656,
+      "learning_rate": 1.3768450184501846e-06,
+      "loss": 1.3894,
+      "step": 1828
+    },
+    {
+      "epoch": 2.2762912258867454,
+      "grad_norm": 3.39282488822937,
+      "learning_rate": 1.374538745387454e-06,
+      "loss": 1.3776,
+      "step": 1829
+    },
+    {
+      "epoch": 2.277535780958307,
+      "grad_norm": 3.250432014465332,
+      "learning_rate": 1.3722324723247232e-06,
+      "loss": 1.3511,
+      "step": 1830
+    },
+    {
+      "epoch": 2.277535780958307,
+      "eval_loss": 1.6178572177886963,
+      "eval_runtime": 45.3086,
+      "eval_samples_per_second": 22.071,
+      "eval_steps_per_second": 0.927,
+      "step": 1830
+    },
+    {
+      "epoch": 2.2787803360298695,
+      "grad_norm": 3.4044647216796875,
+      "learning_rate": 1.3699261992619929e-06,
+      "loss": 1.3548,
+      "step": 1831
+    },
+    {
+      "epoch": 2.2800248911014314,
+      "grad_norm": 3.4996840953826904,
+      "learning_rate": 1.367619926199262e-06,
+      "loss": 1.4069,
+      "step": 1832
+    },
+    {
+      "epoch": 2.281269446172993,
+      "grad_norm": 3.4036638736724854,
+      "learning_rate": 1.3653136531365315e-06,
+      "loss": 1.3499,
+      "step": 1833
+    },
+    {
+      "epoch": 2.282514001244555,
+      "grad_norm": 3.328805685043335,
+      "learning_rate": 1.363007380073801e-06,
+      "loss": 1.3266,
+      "step": 1834
+    },
+    {
+      "epoch": 2.283758556316117,
+      "grad_norm": 3.410924196243286,
+      "learning_rate": 1.3607011070110701e-06,
+      "loss": 1.3303,
+      "step": 1835
+    },
+    {
+      "epoch": 2.2850031113876788,
+      "grad_norm": 3.323162794113159,
+      "learning_rate": 1.3583948339483396e-06,
+      "loss": 1.3284,
+      "step": 1836
+    },
+    {
+      "epoch": 2.286247666459241,
+      "grad_norm": 3.3356032371520996,
+      "learning_rate": 1.356088560885609e-06,
+      "loss": 1.2842,
+      "step": 1837
+    },
+    {
+      "epoch": 2.287492221530803,
+      "grad_norm": 3.3366599082946777,
+      "learning_rate": 1.3537822878228784e-06,
+      "loss": 1.321,
+      "step": 1838
+    },
+    {
+      "epoch": 2.2887367766023647,
+      "grad_norm": 3.3824799060821533,
+      "learning_rate": 1.3514760147601478e-06,
+      "loss": 1.3744,
+      "step": 1839
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "grad_norm": 3.6268486976623535,
+      "learning_rate": 1.349169741697417e-06,
+      "loss": 1.4097,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "eval_loss": 1.6148018836975098,
+      "eval_runtime": 48.3459,
+      "eval_samples_per_second": 20.684,
+      "eval_steps_per_second": 0.869,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2912258867454884,
+      "grad_norm": 3.3530380725860596,
+      "learning_rate": 1.3468634686346865e-06,
+      "loss": 1.3924,
+      "step": 1841
+    },
+    {
+      "epoch": 2.2924704418170503,
+      "grad_norm": 3.456149101257324,
+      "learning_rate": 1.3445571955719559e-06,
+      "loss": 1.3837,
+      "step": 1842
+    },
+    {
+      "epoch": 2.293714996888612,
+      "grad_norm": 3.37180495262146,
+      "learning_rate": 1.3422509225092253e-06,
+      "loss": 1.3411,
+      "step": 1843
+    },
+    {
+      "epoch": 2.2949595519601744,
+      "grad_norm": 3.286172389984131,
+      "learning_rate": 1.3399446494464945e-06,
+      "loss": 1.2888,
+      "step": 1844
+    },
+    {
+      "epoch": 2.2962041070317363,
+      "grad_norm": 3.3966121673583984,
+      "learning_rate": 1.337638376383764e-06,
+      "loss": 1.3644,
+      "step": 1845
+    },
+    {
+      "epoch": 2.297448662103298,
+      "grad_norm": 3.4066531658172607,
+      "learning_rate": 1.3353321033210334e-06,
+      "loss": 1.3543,
+      "step": 1846
+    },
+    {
+      "epoch": 2.29869321717486,
+      "grad_norm": 3.542680501937866,
+      "learning_rate": 1.3330258302583028e-06,
+      "loss": 1.3823,
+      "step": 1847
+    },
+    {
+      "epoch": 2.299937772246422,
+      "grad_norm": 3.323636770248413,
+      "learning_rate": 1.330719557195572e-06,
+      "loss": 1.3031,
+      "step": 1848
+    },
+    {
+      "epoch": 2.3011823273179837,
+      "grad_norm": 3.4802908897399902,
+      "learning_rate": 1.3284132841328414e-06,
+      "loss": 1.4001,
+      "step": 1849
+    },
+    {
+      "epoch": 2.302426882389546,
+      "grad_norm": 3.310987710952759,
+      "learning_rate": 1.3261070110701108e-06,
+      "loss": 1.3385,
+      "step": 1850
+    },
+    {
+      "epoch": 2.302426882389546,
+      "eval_loss": 1.615319848060608,
+      "eval_runtime": 44.8343,
+      "eval_samples_per_second": 22.304,
+      "eval_steps_per_second": 0.937,
+      "step": 1850
+    },
+    {
+      "epoch": 2.303671437461108,
+      "grad_norm": 3.3461661338806152,
+      "learning_rate": 1.3238007380073803e-06,
+      "loss": 1.3461,
+      "step": 1851
+    },
+    {
+      "epoch": 2.3049159925326697,
+      "grad_norm": 3.317770481109619,
+      "learning_rate": 1.3214944649446495e-06,
+      "loss": 1.338,
+      "step": 1852
+    },
+    {
+      "epoch": 2.3061605476042315,
+      "grad_norm": 3.3806679248809814,
+      "learning_rate": 1.319188191881919e-06,
+      "loss": 1.3367,
+      "step": 1853
+    },
+    {
+      "epoch": 2.3074051026757934,
+      "grad_norm": 3.366933822631836,
+      "learning_rate": 1.3168819188191883e-06,
+      "loss": 1.3717,
+      "step": 1854
+    },
+    {
+      "epoch": 2.308649657747355,
+      "grad_norm": 3.3250250816345215,
+      "learning_rate": 1.3145756457564575e-06,
+      "loss": 1.3585,
+      "step": 1855
+    },
+    {
+      "epoch": 2.309894212818917,
+      "grad_norm": 3.320171356201172,
+      "learning_rate": 1.3122693726937272e-06,
+      "loss": 1.3491,
+      "step": 1856
+    },
+    {
+      "epoch": 2.3111387678904793,
+      "grad_norm": 3.418029546737671,
+      "learning_rate": 1.3099630996309964e-06,
+      "loss": 1.3627,
+      "step": 1857
+    },
+    {
+      "epoch": 2.312383322962041,
+      "grad_norm": 3.3762004375457764,
+      "learning_rate": 1.3076568265682658e-06,
+      "loss": 1.3409,
+      "step": 1858
+    },
+    {
+      "epoch": 2.313627878033603,
+      "grad_norm": 3.3099405765533447,
+      "learning_rate": 1.3053505535055352e-06,
+      "loss": 1.3593,
+      "step": 1859
+    },
+    {
+      "epoch": 2.314872433105165,
+      "grad_norm": 3.4060535430908203,
+      "learning_rate": 1.3030442804428044e-06,
+      "loss": 1.3363,
+      "step": 1860
+    },
+    {
+      "epoch": 2.314872433105165,
+      "eval_loss": 1.6176154613494873,
+      "eval_runtime": 46.8022,
+      "eval_samples_per_second": 21.367,
+      "eval_steps_per_second": 0.897,
+      "step": 1860
+    },
+    {
+      "epoch": 2.3161169881767267,
+      "grad_norm": 3.2827699184417725,
+      "learning_rate": 1.3007380073800739e-06,
+      "loss": 1.2718,
+      "step": 1861
+    },
+    {
+      "epoch": 2.3173615432482886,
+      "grad_norm": 3.5225489139556885,
+      "learning_rate": 1.2984317343173433e-06,
+      "loss": 1.3728,
+      "step": 1862
+    },
+    {
+      "epoch": 2.318606098319851,
+      "grad_norm": 3.475665807723999,
+      "learning_rate": 1.2961254612546125e-06,
+      "loss": 1.3772,
+      "step": 1863
+    },
+    {
+      "epoch": 2.3198506533914127,
+      "grad_norm": 3.385239839553833,
+      "learning_rate": 1.2938191881918821e-06,
+      "loss": 1.3471,
+      "step": 1864
+    },
+    {
+      "epoch": 2.3210952084629746,
+      "grad_norm": 3.312469005584717,
+      "learning_rate": 1.2915129151291513e-06,
+      "loss": 1.2961,
+      "step": 1865
+    },
+    {
+      "epoch": 2.3223397635345364,
+      "grad_norm": 3.3883018493652344,
+      "learning_rate": 1.2892066420664208e-06,
+      "loss": 1.3778,
+      "step": 1866
+    },
+    {
+      "epoch": 2.3235843186060983,
+      "grad_norm": 3.368964195251465,
+      "learning_rate": 1.2869003690036902e-06,
+      "loss": 1.3153,
+      "step": 1867
+    },
+    {
+      "epoch": 2.32482887367766,
+      "grad_norm": 3.4132297039031982,
+      "learning_rate": 1.2845940959409594e-06,
+      "loss": 1.3718,
+      "step": 1868
+    },
+    {
+      "epoch": 2.326073428749222,
+      "grad_norm": 3.3750340938568115,
+      "learning_rate": 1.282287822878229e-06,
+      "loss": 1.4035,
+      "step": 1869
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "grad_norm": 3.372588872909546,
+      "learning_rate": 1.2799815498154982e-06,
+      "loss": 1.3591,
+      "step": 1870
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "eval_loss": 1.6196452379226685,
+      "eval_runtime": 43.8125,
+      "eval_samples_per_second": 22.825,
+      "eval_steps_per_second": 0.959,
+      "step": 1870
+    },
+    {
+      "epoch": 2.328562538892346,
+      "grad_norm": 3.341944694519043,
+      "learning_rate": 1.2776752767527675e-06,
+      "loss": 1.3239,
+      "step": 1871
+    },
+    {
+      "epoch": 2.329807093963908,
+      "grad_norm": 3.4855759143829346,
+      "learning_rate": 1.275369003690037e-06,
+      "loss": 1.3712,
+      "step": 1872
+    },
+    {
+      "epoch": 2.33105164903547,
+      "grad_norm": 3.577047348022461,
+      "learning_rate": 1.2730627306273063e-06,
+      "loss": 1.3502,
+      "step": 1873
+    },
+    {
+      "epoch": 2.3322962041070316,
+      "grad_norm": 3.479374885559082,
+      "learning_rate": 1.270756457564576e-06,
+      "loss": 1.3296,
+      "step": 1874
+    },
+    {
+      "epoch": 2.3335407591785935,
+      "grad_norm": 3.2203609943389893,
+      "learning_rate": 1.2684501845018451e-06,
+      "loss": 1.3105,
+      "step": 1875
+    },
+    {
+      "epoch": 2.334785314250156,
+      "grad_norm": 3.3177664279937744,
+      "learning_rate": 1.2661439114391144e-06,
+      "loss": 1.3365,
+      "step": 1876
+    },
+    {
+      "epoch": 2.3360298693217176,
+      "grad_norm": 3.224146842956543,
+      "learning_rate": 1.263837638376384e-06,
+      "loss": 1.3172,
+      "step": 1877
+    },
+    {
+      "epoch": 2.3372744243932795,
+      "grad_norm": 3.336106538772583,
+      "learning_rate": 1.2615313653136532e-06,
+      "loss": 1.3248,
+      "step": 1878
+    },
+    {
+      "epoch": 2.3385189794648413,
+      "grad_norm": 3.4414947032928467,
+      "learning_rate": 1.2592250922509224e-06,
+      "loss": 1.3637,
+      "step": 1879
+    },
+    {
+      "epoch": 2.339763534536403,
+      "grad_norm": 3.5528085231781006,
+      "learning_rate": 1.256918819188192e-06,
+      "loss": 1.3986,
+      "step": 1880
+    },
+    {
+      "epoch": 2.339763534536403,
+      "eval_loss": 1.6137934923171997,
+      "eval_runtime": 43.8859,
+      "eval_samples_per_second": 22.786,
+      "eval_steps_per_second": 0.957,
+      "step": 1880
+    },
+    {
+      "epoch": 2.341008089607965,
+      "grad_norm": 3.406446695327759,
+      "learning_rate": 1.2546125461254613e-06,
+      "loss": 1.3479,
+      "step": 1881
+    },
+    {
+      "epoch": 2.342252644679527,
+      "grad_norm": 3.4129297733306885,
+      "learning_rate": 1.2523062730627309e-06,
+      "loss": 1.3227,
+      "step": 1882
+    },
+    {
+      "epoch": 2.343497199751089,
+      "grad_norm": 3.3013577461242676,
+      "learning_rate": 1.25e-06,
+      "loss": 1.3277,
+      "step": 1883
+    },
+    {
+      "epoch": 2.344741754822651,
+      "grad_norm": 3.3039028644561768,
+      "learning_rate": 1.2476937269372695e-06,
+      "loss": 1.3208,
+      "step": 1884
+    },
+    {
+      "epoch": 2.345986309894213,
+      "grad_norm": 3.3239917755126953,
+      "learning_rate": 1.245387453874539e-06,
+      "loss": 1.3293,
+      "step": 1885
+    },
+    {
+      "epoch": 2.3472308649657747,
+      "grad_norm": 3.239821195602417,
+      "learning_rate": 1.2430811808118082e-06,
+      "loss": 1.3191,
+      "step": 1886
+    },
+    {
+      "epoch": 2.3484754200373366,
+      "grad_norm": 3.6869564056396484,
+      "learning_rate": 1.2407749077490776e-06,
+      "loss": 1.363,
+      "step": 1887
+    },
+    {
+      "epoch": 2.3497199751088984,
+      "grad_norm": 3.5775156021118164,
+      "learning_rate": 1.238468634686347e-06,
+      "loss": 1.3564,
+      "step": 1888
+    },
+    {
+      "epoch": 2.3509645301804607,
+      "grad_norm": 3.3177807331085205,
+      "learning_rate": 1.2361623616236164e-06,
+      "loss": 1.3044,
+      "step": 1889
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "grad_norm": 3.622527837753296,
+      "learning_rate": 1.2338560885608856e-06,
+      "loss": 1.381,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "eval_loss": 1.6165976524353027,
+      "eval_runtime": 44.4026,
+      "eval_samples_per_second": 22.521,
+      "eval_steps_per_second": 0.946,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3534536403235844,
+      "grad_norm": 3.3163886070251465,
+      "learning_rate": 1.231549815498155e-06,
+      "loss": 1.3598,
+      "step": 1891
+    },
+    {
+      "epoch": 2.3546981953951462,
+      "grad_norm": 3.3344099521636963,
+      "learning_rate": 1.2292435424354245e-06,
+      "loss": 1.3733,
+      "step": 1892
+    },
+    {
+      "epoch": 2.355942750466708,
+      "grad_norm": 3.259413480758667,
+      "learning_rate": 1.226937269372694e-06,
+      "loss": 1.3281,
+      "step": 1893
+    },
+    {
+      "epoch": 2.35718730553827,
+      "grad_norm": 3.427147150039673,
+      "learning_rate": 1.2246309963099633e-06,
+      "loss": 1.3484,
+      "step": 1894
+    },
+    {
+      "epoch": 2.358431860609832,
+      "grad_norm": 3.3890154361724854,
+      "learning_rate": 1.2223247232472325e-06,
+      "loss": 1.3768,
+      "step": 1895
+    },
+    {
+      "epoch": 2.359676415681394,
+      "grad_norm": 3.2845499515533447,
+      "learning_rate": 1.220018450184502e-06,
+      "loss": 1.3579,
+      "step": 1896
+    },
+    {
+      "epoch": 2.360920970752956,
+      "grad_norm": 3.3046178817749023,
+      "learning_rate": 1.2177121771217714e-06,
+      "loss": 1.3322,
+      "step": 1897
+    },
+    {
+      "epoch": 2.3621655258245178,
+      "grad_norm": 3.4520273208618164,
+      "learning_rate": 1.2154059040590408e-06,
+      "loss": 1.3515,
+      "step": 1898
+    },
+    {
+      "epoch": 2.3634100808960796,
+      "grad_norm": 3.3665220737457275,
+      "learning_rate": 1.21309963099631e-06,
+      "loss": 1.3236,
+      "step": 1899
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "grad_norm": 3.3551578521728516,
+      "learning_rate": 1.2107933579335794e-06,
+      "loss": 1.3568,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "eval_loss": 1.6159367561340332,
+      "eval_runtime": 44.1216,
+      "eval_samples_per_second": 22.665,
+      "eval_steps_per_second": 0.952,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3658991910392033,
+      "grad_norm": 3.6204397678375244,
+      "learning_rate": 1.2084870848708489e-06,
+      "loss": 1.4072,
+      "step": 1901
+    },
+    {
+      "epoch": 2.3671437461107656,
+      "grad_norm": 3.3525922298431396,
+      "learning_rate": 1.2061808118081183e-06,
+      "loss": 1.328,
+      "step": 1902
+    },
+    {
+      "epoch": 2.3683883011823275,
+      "grad_norm": 3.3686022758483887,
+      "learning_rate": 1.2038745387453875e-06,
+      "loss": 1.3086,
+      "step": 1903
+    },
+    {
+      "epoch": 2.3696328562538893,
+      "grad_norm": 3.5756561756134033,
+      "learning_rate": 1.201568265682657e-06,
+      "loss": 1.3784,
+      "step": 1904
+    },
+    {
+      "epoch": 2.370877411325451,
+      "grad_norm": 3.4186534881591797,
+      "learning_rate": 1.1992619926199263e-06,
+      "loss": 1.3546,
+      "step": 1905
+    },
+    {
+      "epoch": 2.372121966397013,
+      "grad_norm": 3.367868661880493,
+      "learning_rate": 1.1969557195571958e-06,
+      "loss": 1.3527,
+      "step": 1906
+    },
+    {
+      "epoch": 2.373366521468575,
+      "grad_norm": 3.3395912647247314,
+      "learning_rate": 1.1946494464944652e-06,
+      "loss": 1.3366,
+      "step": 1907
+    },
+    {
+      "epoch": 2.3746110765401367,
+      "grad_norm": 3.2967982292175293,
+      "learning_rate": 1.1923431734317344e-06,
+      "loss": 1.3229,
+      "step": 1908
+    },
+    {
+      "epoch": 2.375855631611699,
+      "grad_norm": 3.2422475814819336,
+      "learning_rate": 1.1900369003690038e-06,
+      "loss": 1.3455,
+      "step": 1909
+    },
+    {
+      "epoch": 2.377100186683261,
+      "grad_norm": 3.3781206607818604,
+      "learning_rate": 1.1877306273062733e-06,
+      "loss": 1.3314,
+      "step": 1910
+    },
+    {
+      "epoch": 2.377100186683261,
+      "eval_loss": 1.6179407835006714,
+      "eval_runtime": 42.9089,
+      "eval_samples_per_second": 23.305,
+      "eval_steps_per_second": 0.979,
+      "step": 1910
+    },
+    {
+      "epoch": 2.3783447417548227,
+      "grad_norm": 3.438586711883545,
+      "learning_rate": 1.1854243542435427e-06,
+      "loss": 1.3654,
+      "step": 1911
+    },
+    {
+      "epoch": 2.3795892968263845,
+      "grad_norm": 3.503896474838257,
+      "learning_rate": 1.1831180811808119e-06,
+      "loss": 1.3716,
+      "step": 1912
+    },
+    {
+      "epoch": 2.3808338518979464,
+      "grad_norm": 3.411320686340332,
+      "learning_rate": 1.1808118081180813e-06,
+      "loss": 1.3315,
+      "step": 1913
+    },
+    {
+      "epoch": 2.3820784069695082,
+      "grad_norm": 3.1860697269439697,
+      "learning_rate": 1.1785055350553507e-06,
+      "loss": 1.3016,
+      "step": 1914
+    },
+    {
+      "epoch": 2.3833229620410705,
+      "grad_norm": 3.4494664669036865,
+      "learning_rate": 1.1761992619926202e-06,
+      "loss": 1.3629,
+      "step": 1915
+    },
+    {
+      "epoch": 2.3845675171126324,
+      "grad_norm": 3.303277015686035,
+      "learning_rate": 1.1738929889298894e-06,
+      "loss": 1.3207,
+      "step": 1916
+    },
+    {
+      "epoch": 2.385812072184194,
+      "grad_norm": 3.254411458969116,
+      "learning_rate": 1.1715867158671588e-06,
+      "loss": 1.3477,
+      "step": 1917
+    },
+    {
+      "epoch": 2.387056627255756,
+      "grad_norm": 3.258666753768921,
+      "learning_rate": 1.1692804428044282e-06,
+      "loss": 1.3452,
+      "step": 1918
+    },
+    {
+      "epoch": 2.388301182327318,
+      "grad_norm": 3.3814001083374023,
+      "learning_rate": 1.1669741697416974e-06,
+      "loss": 1.329,
+      "step": 1919
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "grad_norm": 3.367011547088623,
+      "learning_rate": 1.1646678966789668e-06,
+      "loss": 1.3326,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "eval_loss": 1.6158324480056763,
+      "eval_runtime": 45.8088,
+      "eval_samples_per_second": 21.83,
+      "eval_steps_per_second": 0.917,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3907902924704416,
+      "grad_norm": 3.313123941421509,
+      "learning_rate": 1.1623616236162363e-06,
+      "loss": 1.307,
+      "step": 1921
+    },
+    {
+      "epoch": 2.392034847542004,
+      "grad_norm": 3.27423357963562,
+      "learning_rate": 1.1600553505535057e-06,
+      "loss": 1.3249,
+      "step": 1922
+    },
+    {
+      "epoch": 2.3932794026135658,
+      "grad_norm": 3.2963404655456543,
+      "learning_rate": 1.157749077490775e-06,
+      "loss": 1.3394,
+      "step": 1923
+    },
+    {
+      "epoch": 2.3945239576851276,
+      "grad_norm": 3.387673854827881,
+      "learning_rate": 1.1554428044280443e-06,
+      "loss": 1.3586,
+      "step": 1924
+    },
+    {
+      "epoch": 2.3957685127566895,
+      "grad_norm": 3.2588143348693848,
+      "learning_rate": 1.1531365313653137e-06,
+      "loss": 1.317,
+      "step": 1925
+    },
+    {
+      "epoch": 2.3970130678282513,
+      "grad_norm": 3.285653829574585,
+      "learning_rate": 1.1508302583025832e-06,
+      "loss": 1.2847,
+      "step": 1926
+    },
+    {
+      "epoch": 2.398257622899813,
+      "grad_norm": 3.4495935440063477,
+      "learning_rate": 1.1485239852398524e-06,
+      "loss": 1.3815,
+      "step": 1927
+    },
+    {
+      "epoch": 2.3995021779713754,
+      "grad_norm": 3.287186622619629,
+      "learning_rate": 1.1462177121771218e-06,
+      "loss": 1.3048,
+      "step": 1928
+    },
+    {
+      "epoch": 2.4007467330429373,
+      "grad_norm": 3.318042278289795,
+      "learning_rate": 1.1439114391143912e-06,
+      "loss": 1.313,
+      "step": 1929
+    },
+    {
+      "epoch": 2.401991288114499,
+      "grad_norm": 3.2705087661743164,
+      "learning_rate": 1.1416051660516606e-06,
+      "loss": 1.3207,
+      "step": 1930
+    },
+    {
+      "epoch": 2.401991288114499,
+      "eval_loss": 1.613088846206665,
+      "eval_runtime": 46.0901,
+      "eval_samples_per_second": 21.697,
+      "eval_steps_per_second": 0.911,
+      "step": 1930
+    },
+    {
+      "epoch": 2.403235843186061,
+      "grad_norm": 3.307523250579834,
+      "learning_rate": 1.1392988929889299e-06,
+      "loss": 1.3312,
+      "step": 1931
+    },
+    {
+      "epoch": 2.404480398257623,
+      "grad_norm": 3.3040900230407715,
+      "learning_rate": 1.1369926199261993e-06,
+      "loss": 1.3279,
+      "step": 1932
+    },
+    {
+      "epoch": 2.4057249533291847,
+      "grad_norm": 3.232649087905884,
+      "learning_rate": 1.1346863468634687e-06,
+      "loss": 1.314,
+      "step": 1933
+    },
+    {
+      "epoch": 2.4069695084007465,
+      "grad_norm": 3.3581860065460205,
+      "learning_rate": 1.1323800738007381e-06,
+      "loss": 1.3158,
+      "step": 1934
+    },
+    {
+      "epoch": 2.408214063472309,
+      "grad_norm": 3.378774404525757,
+      "learning_rate": 1.1300738007380073e-06,
+      "loss": 1.3252,
+      "step": 1935
+    },
+    {
+      "epoch": 2.4094586185438707,
+      "grad_norm": 3.4180240631103516,
+      "learning_rate": 1.1277675276752768e-06,
+      "loss": 1.3606,
+      "step": 1936
+    },
+    {
+      "epoch": 2.4107031736154325,
+      "grad_norm": 3.410921335220337,
+      "learning_rate": 1.1254612546125462e-06,
+      "loss": 1.3484,
+      "step": 1937
+    },
+    {
+      "epoch": 2.4119477286869944,
+      "grad_norm": 3.4201443195343018,
+      "learning_rate": 1.1231549815498156e-06,
+      "loss": 1.3659,
+      "step": 1938
+    },
+    {
+      "epoch": 2.413192283758556,
+      "grad_norm": 3.381019115447998,
+      "learning_rate": 1.1208487084870848e-06,
+      "loss": 1.3261,
+      "step": 1939
+    },
+    {
+      "epoch": 2.414436838830118,
+      "grad_norm": 3.516425371170044,
+      "learning_rate": 1.1185424354243542e-06,
+      "loss": 1.3618,
+      "step": 1940
+    },
+    {
+      "epoch": 2.414436838830118,
+      "eval_loss": 1.6142985820770264,
+      "eval_runtime": 45.8868,
+      "eval_samples_per_second": 21.793,
+      "eval_steps_per_second": 0.915,
+      "step": 1940
+    },
+    {
+      "epoch": 2.4156813939016804,
+      "grad_norm": 3.5319905281066895,
+      "learning_rate": 1.1162361623616237e-06,
+      "loss": 1.338,
+      "step": 1941
+    },
+    {
+      "epoch": 2.416925948973242,
+      "grad_norm": 3.4355592727661133,
+      "learning_rate": 1.113929889298893e-06,
+      "loss": 1.3389,
+      "step": 1942
+    },
+    {
+      "epoch": 2.418170504044804,
+      "grad_norm": 3.362187623977661,
+      "learning_rate": 1.1116236162361623e-06,
+      "loss": 1.3194,
+      "step": 1943
+    },
+    {
+      "epoch": 2.419415059116366,
+      "grad_norm": 3.4991087913513184,
+      "learning_rate": 1.1093173431734317e-06,
+      "loss": 1.3758,
+      "step": 1944
+    },
+    {
+      "epoch": 2.4206596141879277,
+      "grad_norm": 3.5951290130615234,
+      "learning_rate": 1.1070110701107011e-06,
+      "loss": 1.3548,
+      "step": 1945
+    },
+    {
+      "epoch": 2.4219041692594896,
+      "grad_norm": 3.3458032608032227,
+      "learning_rate": 1.1047047970479706e-06,
+      "loss": 1.3483,
+      "step": 1946
+    },
+    {
+      "epoch": 2.4231487243310514,
+      "grad_norm": 3.426600933074951,
+      "learning_rate": 1.10239852398524e-06,
+      "loss": 1.3548,
+      "step": 1947
+    },
+    {
+      "epoch": 2.4243932794026137,
+      "grad_norm": 3.5074410438537598,
+      "learning_rate": 1.1000922509225092e-06,
+      "loss": 1.3608,
+      "step": 1948
+    },
+    {
+      "epoch": 2.4256378344741756,
+      "grad_norm": 3.5622129440307617,
+      "learning_rate": 1.0977859778597786e-06,
+      "loss": 1.3861,
+      "step": 1949
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "grad_norm": 3.537771463394165,
+      "learning_rate": 1.095479704797048e-06,
+      "loss": 1.3408,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "eval_loss": 1.612324833869934,
+      "eval_runtime": 46.7327,
+      "eval_samples_per_second": 21.398,
+      "eval_steps_per_second": 0.899,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4281269446172993,
+      "grad_norm": 3.430544137954712,
+      "learning_rate": 1.0931734317343175e-06,
+      "loss": 1.3678,
+      "step": 1951
+    },
+    {
+      "epoch": 2.429371499688861,
+      "grad_norm": 3.2243361473083496,
+      "learning_rate": 1.0908671586715867e-06,
+      "loss": 1.2982,
+      "step": 1952
+    },
+    {
+      "epoch": 2.430616054760423,
+      "grad_norm": 3.426236867904663,
+      "learning_rate": 1.088560885608856e-06,
+      "loss": 1.3827,
+      "step": 1953
+    },
+    {
+      "epoch": 2.4318606098319853,
+      "grad_norm": 3.3180603981018066,
+      "learning_rate": 1.0862546125461255e-06,
+      "loss": 1.3065,
+      "step": 1954
+    },
+    {
+      "epoch": 2.433105164903547,
+      "grad_norm": 3.3775084018707275,
+      "learning_rate": 1.083948339483395e-06,
+      "loss": 1.2759,
+      "step": 1955
+    },
+    {
+      "epoch": 2.434349719975109,
+      "grad_norm": 3.4124209880828857,
+      "learning_rate": 1.0816420664206644e-06,
+      "loss": 1.3571,
+      "step": 1956
+    },
+    {
+      "epoch": 2.435594275046671,
+      "grad_norm": 3.494408369064331,
+      "learning_rate": 1.0793357933579336e-06,
+      "loss": 1.3931,
+      "step": 1957
+    },
+    {
+      "epoch": 2.4368388301182327,
+      "grad_norm": 3.2699358463287354,
+      "learning_rate": 1.077029520295203e-06,
+      "loss": 1.3182,
+      "step": 1958
+    },
+    {
+      "epoch": 2.4380833851897945,
+      "grad_norm": 3.281592607498169,
+      "learning_rate": 1.0747232472324724e-06,
+      "loss": 1.2629,
+      "step": 1959
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "grad_norm": 3.3045406341552734,
+      "learning_rate": 1.0724169741697419e-06,
+      "loss": 1.3093,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "eval_loss": 1.6135376691818237,
+      "eval_runtime": 51.0587,
+      "eval_samples_per_second": 19.585,
+      "eval_steps_per_second": 0.823,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4405724953329186,
+      "grad_norm": 3.3883392810821533,
+      "learning_rate": 1.070110701107011e-06,
+      "loss": 1.4005,
+      "step": 1961
+    },
+    {
+      "epoch": 2.4418170504044805,
+      "grad_norm": 3.3957419395446777,
+      "learning_rate": 1.0678044280442805e-06,
+      "loss": 1.3342,
+      "step": 1962
+    },
+    {
+      "epoch": 2.4430616054760423,
+      "grad_norm": 3.1982152462005615,
+      "learning_rate": 1.06549815498155e-06,
+      "loss": 1.3273,
+      "step": 1963
+    },
+    {
+      "epoch": 2.444306160547604,
+      "grad_norm": 3.3369827270507812,
+      "learning_rate": 1.0631918819188193e-06,
+      "loss": 1.312,
+      "step": 1964
+    },
+    {
+      "epoch": 2.445550715619166,
+      "grad_norm": 3.3651299476623535,
+      "learning_rate": 1.0608856088560888e-06,
+      "loss": 1.3728,
+      "step": 1965
+    },
+    {
+      "epoch": 2.446795270690728,
+      "grad_norm": 3.3393259048461914,
+      "learning_rate": 1.058579335793358e-06,
+      "loss": 1.3454,
+      "step": 1966
+    },
+    {
+      "epoch": 2.44803982576229,
+      "grad_norm": 3.4974067211151123,
+      "learning_rate": 1.0562730627306274e-06,
+      "loss": 1.3747,
+      "step": 1967
+    },
+    {
+      "epoch": 2.449284380833852,
+      "grad_norm": 3.4064528942108154,
+      "learning_rate": 1.0539667896678968e-06,
+      "loss": 1.3695,
+      "step": 1968
+    },
+    {
+      "epoch": 2.450528935905414,
+      "grad_norm": 3.347919225692749,
+      "learning_rate": 1.0516605166051662e-06,
+      "loss": 1.3439,
+      "step": 1969
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "grad_norm": 3.4505932331085205,
+      "learning_rate": 1.0493542435424354e-06,
+      "loss": 1.3723,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "eval_loss": 1.611203908920288,
+      "eval_runtime": 52.5131,
+      "eval_samples_per_second": 19.043,
+      "eval_steps_per_second": 0.8,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4530180460485376,
+      "grad_norm": 3.4287805557250977,
+      "learning_rate": 1.0470479704797049e-06,
+      "loss": 1.3426,
+      "step": 1971
+    },
+    {
+      "epoch": 2.4542626011200994,
+      "grad_norm": 3.4337522983551025,
+      "learning_rate": 1.0447416974169743e-06,
+      "loss": 1.3451,
+      "step": 1972
+    },
+    {
+      "epoch": 2.4555071561916613,
+      "grad_norm": 3.4307212829589844,
+      "learning_rate": 1.0424354243542437e-06,
+      "loss": 1.3097,
+      "step": 1973
+    },
+    {
+      "epoch": 2.4567517112632236,
+      "grad_norm": 3.307767868041992,
+      "learning_rate": 1.0401291512915131e-06,
+      "loss": 1.2751,
+      "step": 1974
+    },
+    {
+      "epoch": 2.4579962663347854,
+      "grad_norm": 3.4809703826904297,
+      "learning_rate": 1.0378228782287824e-06,
+      "loss": 1.3522,
+      "step": 1975
+    },
+    {
+      "epoch": 2.4592408214063473,
+      "grad_norm": 3.235170602798462,
+      "learning_rate": 1.0355166051660518e-06,
+      "loss": 1.3003,
+      "step": 1976
+    },
+    {
+      "epoch": 2.460485376477909,
+      "grad_norm": 3.2969000339508057,
+      "learning_rate": 1.0332103321033212e-06,
+      "loss": 1.2993,
+      "step": 1977
+    },
+    {
+      "epoch": 2.461729931549471,
+      "grad_norm": 3.311535358428955,
+      "learning_rate": 1.0309040590405906e-06,
+      "loss": 1.3262,
+      "step": 1978
+    },
+    {
+      "epoch": 2.462974486621033,
+      "grad_norm": 3.2320244312286377,
+      "learning_rate": 1.0285977859778598e-06,
+      "loss": 1.3111,
+      "step": 1979
+    },
+    {
+      "epoch": 2.464219041692595,
+      "grad_norm": 3.2673771381378174,
+      "learning_rate": 1.0262915129151293e-06,
+      "loss": 1.3439,
+      "step": 1980
+    },
+    {
+      "epoch": 2.464219041692595,
+      "eval_loss": 1.6103569269180298,
+      "eval_runtime": 47.2563,
+      "eval_samples_per_second": 21.161,
+      "eval_steps_per_second": 0.889,
+      "step": 1980
+    },
+    {
+      "epoch": 2.465463596764157,
+      "grad_norm": 3.324113130569458,
+      "learning_rate": 1.0239852398523987e-06,
+      "loss": 1.3278,
+      "step": 1981
+    },
+    {
+      "epoch": 2.466708151835719,
+      "grad_norm": 3.3066444396972656,
+      "learning_rate": 1.021678966789668e-06,
+      "loss": 1.3438,
+      "step": 1982
+    },
+    {
+      "epoch": 2.4679527069072806,
+      "grad_norm": 3.463088035583496,
+      "learning_rate": 1.0193726937269373e-06,
+      "loss": 1.3293,
+      "step": 1983
+    },
+    {
+      "epoch": 2.4691972619788425,
+      "grad_norm": 3.406008005142212,
+      "learning_rate": 1.0170664206642067e-06,
+      "loss": 1.4026,
+      "step": 1984
+    },
+    {
+      "epoch": 2.4704418170504043,
+      "grad_norm": 3.408823251724243,
+      "learning_rate": 1.0147601476014762e-06,
+      "loss": 1.3226,
+      "step": 1985
+    },
+    {
+      "epoch": 2.471686372121966,
+      "grad_norm": 3.2991373538970947,
+      "learning_rate": 1.0124538745387456e-06,
+      "loss": 1.2682,
+      "step": 1986
+    },
+    {
+      "epoch": 2.4729309271935285,
+      "grad_norm": 3.3868234157562256,
+      "learning_rate": 1.010147601476015e-06,
+      "loss": 1.3286,
+      "step": 1987
+    },
+    {
+      "epoch": 2.4741754822650903,
+      "grad_norm": 3.3641555309295654,
+      "learning_rate": 1.0078413284132842e-06,
+      "loss": 1.3423,
+      "step": 1988
+    },
+    {
+      "epoch": 2.475420037336652,
+      "grad_norm": 3.3237619400024414,
+      "learning_rate": 1.0055350553505536e-06,
+      "loss": 1.2979,
+      "step": 1989
+    },
+    {
+      "epoch": 2.476664592408214,
+      "grad_norm": 3.6278553009033203,
+      "learning_rate": 1.003228782287823e-06,
+      "loss": 1.3543,
+      "step": 1990
+    },
+    {
+      "epoch": 2.476664592408214,
+      "eval_loss": 1.6103031635284424,
+      "eval_runtime": 46.0573,
+      "eval_samples_per_second": 21.712,
+      "eval_steps_per_second": 0.912,
+      "step": 1990
+    },
+    {
+      "epoch": 2.477909147479776,
+      "grad_norm": 3.2861995697021484,
+      "learning_rate": 1.0009225092250923e-06,
+      "loss": 1.3303,
+      "step": 1991
+    },
+    {
+      "epoch": 2.4791537025513377,
+      "grad_norm": 3.352261781692505,
+      "learning_rate": 9.986162361623617e-07,
+      "loss": 1.3527,
+      "step": 1992
+    },
+    {
+      "epoch": 2.4803982576229,
+      "grad_norm": 3.483726978302002,
+      "learning_rate": 9.963099630996311e-07,
+      "loss": 1.3762,
+      "step": 1993
+    },
+    {
+      "epoch": 2.481642812694462,
+      "grad_norm": 3.357154607772827,
+      "learning_rate": 9.940036900369005e-07,
+      "loss": 1.3601,
+      "step": 1994
+    },
+    {
+      "epoch": 2.4828873677660237,
+      "grad_norm": 3.4477577209472656,
+      "learning_rate": 9.916974169741698e-07,
+      "loss": 1.3636,
+      "step": 1995
+    },
+    {
+      "epoch": 2.4841319228375855,
+      "grad_norm": 3.377279043197632,
+      "learning_rate": 9.893911439114392e-07,
+      "loss": 1.3349,
+      "step": 1996
+    },
+    {
+      "epoch": 2.4853764779091474,
+      "grad_norm": 3.2705130577087402,
+      "learning_rate": 9.870848708487086e-07,
+      "loss": 1.3659,
+      "step": 1997
+    },
+    {
+      "epoch": 2.4866210329807092,
+      "grad_norm": 3.418225049972534,
+      "learning_rate": 9.84778597785978e-07,
+      "loss": 1.3534,
+      "step": 1998
+    },
+    {
+      "epoch": 2.487865588052271,
+      "grad_norm": 3.2288014888763428,
+      "learning_rate": 9.824723247232472e-07,
+      "loss": 1.3392,
+      "step": 1999
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "grad_norm": 3.3513882160186768,
+      "learning_rate": 9.801660516605167e-07,
+      "loss": 1.3483,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "eval_loss": 1.6108943223953247,
+      "eval_runtime": 48.275,
+      "eval_samples_per_second": 20.715,
+      "eval_steps_per_second": 0.87,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4903546981953952,
+      "grad_norm": 3.35583758354187,
+      "learning_rate": 9.77859778597786e-07,
+      "loss": 1.3808,
+      "step": 2001
+    },
+    {
+      "epoch": 2.491599253266957,
+      "grad_norm": 3.4611411094665527,
+      "learning_rate": 9.755535055350555e-07,
+      "loss": 1.3512,
+      "step": 2002
+    },
+    {
+      "epoch": 2.492843808338519,
+      "grad_norm": 3.389965057373047,
+      "learning_rate": 9.732472324723247e-07,
+      "loss": 1.2777,
+      "step": 2003
+    },
+    {
+      "epoch": 2.4940883634100808,
+      "grad_norm": 3.370044231414795,
+      "learning_rate": 9.709409594095941e-07,
+      "loss": 1.3407,
+      "step": 2004
+    },
+    {
+      "epoch": 2.4953329184816426,
+      "grad_norm": 3.4178175926208496,
+      "learning_rate": 9.686346863468636e-07,
+      "loss": 1.3419,
+      "step": 2005
+    },
+    {
+      "epoch": 2.496577473553205,
+      "grad_norm": 3.4306576251983643,
+      "learning_rate": 9.66328413284133e-07,
+      "loss": 1.3029,
+      "step": 2006
+    },
+    {
+      "epoch": 2.4978220286247668,
+      "grad_norm": 3.4822888374328613,
+      "learning_rate": 9.640221402214022e-07,
+      "loss": 1.3512,
+      "step": 2007
+    },
+    {
+      "epoch": 2.4990665836963286,
+      "grad_norm": 3.2686867713928223,
+      "learning_rate": 9.617158671586716e-07,
+      "loss": 1.3366,
+      "step": 2008
+    },
+    {
+      "epoch": 2.5003111387678905,
+      "grad_norm": 3.413224220275879,
+      "learning_rate": 9.59409594095941e-07,
+      "loss": 1.324,
+      "step": 2009
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "grad_norm": 3.180180311203003,
+      "learning_rate": 9.571033210332105e-07,
+      "loss": 1.3054,
+      "step": 2010
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "eval_loss": 1.6145573854446411,
+      "eval_runtime": 42.6598,
+      "eval_samples_per_second": 23.441,
+      "eval_steps_per_second": 0.985,
+      "step": 2010
+    },
+    {
+      "epoch": 2.502800248911014,
+      "grad_norm": 3.4968016147613525,
+      "learning_rate": 9.547970479704797e-07,
+      "loss": 1.3782,
+      "step": 2011
+    },
+    {
+      "epoch": 2.504044803982576,
+      "grad_norm": 3.270414113998413,
+      "learning_rate": 9.524907749077492e-07,
+      "loss": 1.3389,
+      "step": 2012
+    },
+    {
+      "epoch": 2.5052893590541383,
+      "grad_norm": 3.3850443363189697,
+      "learning_rate": 9.501845018450185e-07,
+      "loss": 1.3335,
+      "step": 2013
+    },
+    {
+      "epoch": 2.5065339141257,
+      "grad_norm": 3.3836278915405273,
+      "learning_rate": 9.478782287822879e-07,
+      "loss": 1.3343,
+      "step": 2014
+    },
+    {
+      "epoch": 2.507778469197262,
+      "grad_norm": 3.4950475692749023,
+      "learning_rate": 9.455719557195573e-07,
+      "loss": 1.337,
+      "step": 2015
+    },
+    {
+      "epoch": 2.509023024268824,
+      "grad_norm": 3.368262529373169,
+      "learning_rate": 9.432656826568267e-07,
+      "loss": 1.32,
+      "step": 2016
+    },
+    {
+      "epoch": 2.5102675793403857,
+      "grad_norm": 3.4199533462524414,
+      "learning_rate": 9.40959409594096e-07,
+      "loss": 1.3754,
+      "step": 2017
+    },
+    {
+      "epoch": 2.511512134411948,
+      "grad_norm": 3.3372180461883545,
+      "learning_rate": 9.386531365313654e-07,
+      "loss": 1.3319,
+      "step": 2018
+    },
+    {
+      "epoch": 2.51275668948351,
+      "grad_norm": 3.3614137172698975,
+      "learning_rate": 9.363468634686347e-07,
+      "loss": 1.383,
+      "step": 2019
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "grad_norm": 3.4617390632629395,
+      "learning_rate": 9.340405904059042e-07,
+      "loss": 1.3628,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "eval_loss": 1.6153606176376343,
+      "eval_runtime": 46.8889,
+      "eval_samples_per_second": 21.327,
+      "eval_steps_per_second": 0.896,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5152457996266335,
+      "grad_norm": 3.3711016178131104,
+      "learning_rate": 9.317343173431735e-07,
+      "loss": 1.3449,
+      "step": 2021
+    },
+    {
+      "epoch": 2.5164903546981954,
+      "grad_norm": 3.2858471870422363,
+      "learning_rate": 9.294280442804429e-07,
+      "loss": 1.348,
+      "step": 2022
+    },
+    {
+      "epoch": 2.5177349097697572,
+      "grad_norm": 3.445166826248169,
+      "learning_rate": 9.271217712177122e-07,
+      "loss": 1.3182,
+      "step": 2023
+    },
+    {
+      "epoch": 2.518979464841319,
+      "grad_norm": 3.417206287384033,
+      "learning_rate": 9.248154981549816e-07,
+      "loss": 1.3241,
+      "step": 2024
+    },
+    {
+      "epoch": 2.520224019912881,
+      "grad_norm": 3.333965539932251,
+      "learning_rate": 9.22509225092251e-07,
+      "loss": 1.3214,
+      "step": 2025
+    },
+    {
+      "epoch": 2.521468574984443,
+      "grad_norm": 3.4090497493743896,
+      "learning_rate": 9.202029520295204e-07,
+      "loss": 1.3566,
+      "step": 2026
+    },
+    {
+      "epoch": 2.522713130056005,
+      "grad_norm": 3.312790870666504,
+      "learning_rate": 9.178966789667898e-07,
+      "loss": 1.3181,
+      "step": 2027
+    },
+    {
+      "epoch": 2.523957685127567,
+      "grad_norm": 3.254122734069824,
+      "learning_rate": 9.155904059040591e-07,
+      "loss": 1.3079,
+      "step": 2028
+    },
+    {
+      "epoch": 2.5252022401991288,
+      "grad_norm": 3.5144238471984863,
+      "learning_rate": 9.132841328413284e-07,
+      "loss": 1.3586,
+      "step": 2029
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "grad_norm": 3.6083173751831055,
+      "learning_rate": 9.109778597785979e-07,
+      "loss": 1.3833,
+      "step": 2030
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "eval_loss": 1.6153796911239624,
+      "eval_runtime": 52.1312,
+      "eval_samples_per_second": 19.182,
+      "eval_steps_per_second": 0.806,
+      "step": 2030
+    },
+    {
+      "epoch": 2.527691350342253,
+      "grad_norm": 3.489710569381714,
+      "learning_rate": 9.086715867158673e-07,
+      "loss": 1.3466,
+      "step": 2031
+    },
+    {
+      "epoch": 2.5289359054138147,
+      "grad_norm": 3.4363298416137695,
+      "learning_rate": 9.063653136531365e-07,
+      "loss": 1.3461,
+      "step": 2032
+    },
+    {
+      "epoch": 2.5301804604853766,
+      "grad_norm": 3.627476930618286,
+      "learning_rate": 9.040590405904059e-07,
+      "loss": 1.3655,
+      "step": 2033
+    },
+    {
+      "epoch": 2.5314250155569384,
+      "grad_norm": 3.2995362281799316,
+      "learning_rate": 9.017527675276753e-07,
+      "loss": 1.297,
+      "step": 2034
+    },
+    {
+      "epoch": 2.5326695706285003,
+      "grad_norm": 3.4116721153259277,
+      "learning_rate": 8.994464944649448e-07,
+      "loss": 1.3319,
+      "step": 2035
+    },
+    {
+      "epoch": 2.533914125700062,
+      "grad_norm": 3.4598546028137207,
+      "learning_rate": 8.971402214022142e-07,
+      "loss": 1.3646,
+      "step": 2036
+    },
+    {
+      "epoch": 2.535158680771624,
+      "grad_norm": 3.4111695289611816,
+      "learning_rate": 8.948339483394834e-07,
+      "loss": 1.322,
+      "step": 2037
+    },
+    {
+      "epoch": 2.536403235843186,
+      "grad_norm": 3.371891736984253,
+      "learning_rate": 8.925276752767528e-07,
+      "loss": 1.362,
+      "step": 2038
+    },
+    {
+      "epoch": 2.537647790914748,
+      "grad_norm": 3.4267642498016357,
+      "learning_rate": 8.902214022140222e-07,
+      "loss": 1.3829,
+      "step": 2039
+    },
+    {
+      "epoch": 2.53889234598631,
+      "grad_norm": 3.2384777069091797,
+      "learning_rate": 8.879151291512917e-07,
+      "loss": 1.2771,
+      "step": 2040
+    },
+    {
+      "epoch": 2.53889234598631,
+      "eval_loss": 1.6134564876556396,
+      "eval_runtime": 49.4544,
+      "eval_samples_per_second": 20.221,
+      "eval_steps_per_second": 0.849,
+      "step": 2040
+    },
+    {
+      "epoch": 2.540136901057872,
+      "grad_norm": 3.3554086685180664,
+      "learning_rate": 8.856088560885609e-07,
+      "loss": 1.327,
+      "step": 2041
+    },
+    {
+      "epoch": 2.5413814561294337,
+      "grad_norm": 3.297962188720703,
+      "learning_rate": 8.833025830258303e-07,
+      "loss": 1.3257,
+      "step": 2042
+    },
+    {
+      "epoch": 2.5426260112009955,
+      "grad_norm": 3.4998221397399902,
+      "learning_rate": 8.809963099630997e-07,
+      "loss": 1.3782,
+      "step": 2043
+    },
+    {
+      "epoch": 2.543870566272558,
+      "grad_norm": 3.3661816120147705,
+      "learning_rate": 8.786900369003691e-07,
+      "loss": 1.3338,
+      "step": 2044
+    },
+    {
+      "epoch": 2.5451151213441197,
+      "grad_norm": 3.7328436374664307,
+      "learning_rate": 8.763837638376385e-07,
+      "loss": 1.4149,
+      "step": 2045
+    },
+    {
+      "epoch": 2.5463596764156815,
+      "grad_norm": 3.621159315109253,
+      "learning_rate": 8.740774907749078e-07,
+      "loss": 1.3104,
+      "step": 2046
+    },
+    {
+      "epoch": 2.5476042314872434,
+      "grad_norm": 3.5403215885162354,
+      "learning_rate": 8.717712177121772e-07,
+      "loss": 1.3278,
+      "step": 2047
+    },
+    {
+      "epoch": 2.548848786558805,
+      "grad_norm": 3.3124406337738037,
+      "learning_rate": 8.694649446494466e-07,
+      "loss": 1.3313,
+      "step": 2048
+    },
+    {
+      "epoch": 2.550093341630367,
+      "grad_norm": 3.4480574131011963,
+      "learning_rate": 8.671586715867159e-07,
+      "loss": 1.3225,
+      "step": 2049
+    },
+    {
+      "epoch": 2.551337896701929,
+      "grad_norm": 3.372326135635376,
+      "learning_rate": 8.648523985239853e-07,
+      "loss": 1.3377,
+      "step": 2050
+    },
+    {
+      "epoch": 2.551337896701929,
+      "eval_loss": 1.6056281328201294,
+      "eval_runtime": 52.0326,
+      "eval_samples_per_second": 19.219,
+      "eval_steps_per_second": 0.807,
+      "step": 2050
+    },
+    {
+      "epoch": 2.5525824517734907,
+      "grad_norm": 3.4119911193847656,
+      "learning_rate": 8.625461254612547e-07,
+      "loss": 1.3462,
+      "step": 2051
+    },
+    {
+      "epoch": 2.553827006845053,
+      "grad_norm": 3.5083751678466797,
+      "learning_rate": 8.602398523985241e-07,
+      "loss": 1.3632,
+      "step": 2052
+    },
+    {
+      "epoch": 2.555071561916615,
+      "grad_norm": 3.279911518096924,
+      "learning_rate": 8.579335793357934e-07,
+      "loss": 1.3326,
+      "step": 2053
+    },
+    {
+      "epoch": 2.5563161169881767,
+      "grad_norm": 3.2245192527770996,
+      "learning_rate": 8.556273062730628e-07,
+      "loss": 1.3033,
+      "step": 2054
+    },
+    {
+      "epoch": 2.5575606720597386,
+      "grad_norm": 3.3703253269195557,
+      "learning_rate": 8.533210332103322e-07,
+      "loss": 1.3219,
+      "step": 2055
+    },
+    {
+      "epoch": 2.5588052271313004,
+      "grad_norm": 3.3931314945220947,
+      "learning_rate": 8.510147601476016e-07,
+      "loss": 1.2992,
+      "step": 2056
+    },
+    {
+      "epoch": 2.5600497822028627,
+      "grad_norm": 3.5495870113372803,
+      "learning_rate": 8.487084870848709e-07,
+      "loss": 1.3872,
+      "step": 2057
+    },
+    {
+      "epoch": 2.5612943372744246,
+      "grad_norm": 3.476536273956299,
+      "learning_rate": 8.464022140221403e-07,
+      "loss": 1.3609,
+      "step": 2058
+    },
+    {
+      "epoch": 2.5625388923459864,
+      "grad_norm": 3.4503397941589355,
+      "learning_rate": 8.440959409594096e-07,
+      "loss": 1.339,
+      "step": 2059
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "grad_norm": 3.3848440647125244,
+      "learning_rate": 8.417896678966791e-07,
+      "loss": 1.3369,
+      "step": 2060
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "eval_loss": 1.61435866355896,
+      "eval_runtime": 48.4317,
+      "eval_samples_per_second": 20.648,
+      "eval_steps_per_second": 0.867,
+      "step": 2060
+    },
+    {
+      "epoch": 2.56502800248911,
+      "grad_norm": 3.474571466445923,
+      "learning_rate": 8.394833948339484e-07,
+      "loss": 1.379,
+      "step": 2061
+    },
+    {
+      "epoch": 2.566272557560672,
+      "grad_norm": 3.487356185913086,
+      "learning_rate": 8.371771217712178e-07,
+      "loss": 1.365,
+      "step": 2062
+    },
+    {
+      "epoch": 2.567517112632234,
+      "grad_norm": 3.5649149417877197,
+      "learning_rate": 8.348708487084871e-07,
+      "loss": 1.3619,
+      "step": 2063
+    },
+    {
+      "epoch": 2.5687616677037957,
+      "grad_norm": 3.572577714920044,
+      "learning_rate": 8.325645756457565e-07,
+      "loss": 1.3877,
+      "step": 2064
+    },
+    {
+      "epoch": 2.570006222775358,
+      "grad_norm": 3.3802125453948975,
+      "learning_rate": 8.302583025830259e-07,
+      "loss": 1.3323,
+      "step": 2065
+    },
+    {
+      "epoch": 2.57125077784692,
+      "grad_norm": 3.317291498184204,
+      "learning_rate": 8.279520295202953e-07,
+      "loss": 1.3532,
+      "step": 2066
+    },
+    {
+      "epoch": 2.5724953329184816,
+      "grad_norm": 3.485429525375366,
+      "learning_rate": 8.256457564575647e-07,
+      "loss": 1.3561,
+      "step": 2067
+    },
+    {
+      "epoch": 2.5737398879900435,
+      "grad_norm": 3.4251649379730225,
+      "learning_rate": 8.233394833948339e-07,
+      "loss": 1.3494,
+      "step": 2068
+    },
+    {
+      "epoch": 2.5749844430616053,
+      "grad_norm": 3.3372538089752197,
+      "learning_rate": 8.210332103321033e-07,
+      "loss": 1.3116,
+      "step": 2069
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "grad_norm": 3.3535027503967285,
+      "learning_rate": 8.187269372693728e-07,
+      "loss": 1.3541,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "eval_loss": 1.6068458557128906,
+      "eval_runtime": 51.6237,
+      "eval_samples_per_second": 19.371,
+      "eval_steps_per_second": 0.814,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5774735532047295,
+      "grad_norm": 3.4335787296295166,
+      "learning_rate": 8.164206642066422e-07,
+      "loss": 1.3884,
+      "step": 2071
+    },
+    {
+      "epoch": 2.5787181082762913,
+      "grad_norm": 3.436685800552368,
+      "learning_rate": 8.141143911439114e-07,
+      "loss": 1.3508,
+      "step": 2072
+    },
+    {
+      "epoch": 2.579962663347853,
+      "grad_norm": 3.333679676055908,
+      "learning_rate": 8.118081180811808e-07,
+      "loss": 1.3485,
+      "step": 2073
+    },
+    {
+      "epoch": 2.581207218419415,
+      "grad_norm": 3.4076309204101562,
+      "learning_rate": 8.095018450184502e-07,
+      "loss": 1.3219,
+      "step": 2074
+    },
+    {
+      "epoch": 2.582451773490977,
+      "grad_norm": 3.3851561546325684,
+      "learning_rate": 8.071955719557197e-07,
+      "loss": 1.3365,
+      "step": 2075
+    },
+    {
+      "epoch": 2.5836963285625387,
+      "grad_norm": 3.3714189529418945,
+      "learning_rate": 8.048892988929891e-07,
+      "loss": 1.3448,
+      "step": 2076
+    },
+    {
+      "epoch": 2.5849408836341006,
+      "grad_norm": 3.32585072517395,
+      "learning_rate": 8.025830258302583e-07,
+      "loss": 1.338,
+      "step": 2077
+    },
+    {
+      "epoch": 2.586185438705663,
+      "grad_norm": 3.4017553329467773,
+      "learning_rate": 8.002767527675277e-07,
+      "loss": 1.3597,
+      "step": 2078
+    },
+    {
+      "epoch": 2.5874299937772247,
+      "grad_norm": 3.334178924560547,
+      "learning_rate": 7.979704797047971e-07,
+      "loss": 1.2981,
+      "step": 2079
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "grad_norm": 3.443598985671997,
+      "learning_rate": 7.956642066420666e-07,
+      "loss": 1.3188,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "eval_loss": 1.6108036041259766,
+      "eval_runtime": 50.3752,
+      "eval_samples_per_second": 19.851,
+      "eval_steps_per_second": 0.834,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5899191039203484,
+      "grad_norm": 3.460387706756592,
+      "learning_rate": 7.933579335793358e-07,
+      "loss": 1.3794,
+      "step": 2081
+    },
+    {
+      "epoch": 2.5911636589919103,
+      "grad_norm": 3.370560646057129,
+      "learning_rate": 7.910516605166052e-07,
+      "loss": 1.3068,
+      "step": 2082
+    },
+    {
+      "epoch": 2.5924082140634725,
+      "grad_norm": 3.55534291267395,
+      "learning_rate": 7.887453874538746e-07,
+      "loss": 1.3581,
+      "step": 2083
+    },
+    {
+      "epoch": 2.5936527691350344,
+      "grad_norm": 3.322735548019409,
+      "learning_rate": 7.86439114391144e-07,
+      "loss": 1.3399,
+      "step": 2084
+    },
+    {
+      "epoch": 2.5948973242065962,
+      "grad_norm": 3.5256693363189697,
+      "learning_rate": 7.841328413284134e-07,
+      "loss": 1.377,
+      "step": 2085
+    },
+    {
+      "epoch": 2.596141879278158,
+      "grad_norm": 3.4358303546905518,
+      "learning_rate": 7.818265682656827e-07,
+      "loss": 1.3486,
+      "step": 2086
+    },
+    {
+      "epoch": 2.59738643434972,
+      "grad_norm": 3.401855230331421,
+      "learning_rate": 7.795202952029521e-07,
+      "loss": 1.3013,
+      "step": 2087
+    },
+    {
+      "epoch": 2.598630989421282,
+      "grad_norm": 3.414670467376709,
+      "learning_rate": 7.772140221402215e-07,
+      "loss": 1.3342,
+      "step": 2088
+    },
+    {
+      "epoch": 2.5998755444928436,
+      "grad_norm": 3.3348028659820557,
+      "learning_rate": 7.749077490774908e-07,
+      "loss": 1.3594,
+      "step": 2089
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "grad_norm": 3.4095754623413086,
+      "learning_rate": 7.726014760147602e-07,
+      "loss": 1.3739,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "eval_loss": 1.6106606721878052,
+      "eval_runtime": 49.3878,
+      "eval_samples_per_second": 20.248,
+      "eval_steps_per_second": 0.85,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6023646546359678,
+      "grad_norm": 3.247753143310547,
+      "learning_rate": 7.702952029520296e-07,
+      "loss": 1.3026,
+      "step": 2091
+    },
+    {
+      "epoch": 2.6036092097075296,
+      "grad_norm": 3.3973324298858643,
+      "learning_rate": 7.67988929889299e-07,
+      "loss": 1.3381,
+      "step": 2092
+    },
+    {
+      "epoch": 2.6048537647790915,
+      "grad_norm": 3.4262280464172363,
+      "learning_rate": 7.656826568265683e-07,
+      "loss": 1.3595,
+      "step": 2093
+    },
+    {
+      "epoch": 2.6060983198506533,
+      "grad_norm": 3.401784658432007,
+      "learning_rate": 7.633763837638377e-07,
+      "loss": 1.3339,
+      "step": 2094
+    },
+    {
+      "epoch": 2.607342874922215,
+      "grad_norm": 3.3231465816497803,
+      "learning_rate": 7.610701107011071e-07,
+      "loss": 1.2529,
+      "step": 2095
+    },
+    {
+      "epoch": 2.6085874299937775,
+      "grad_norm": 3.412613868713379,
+      "learning_rate": 7.587638376383765e-07,
+      "loss": 1.3341,
+      "step": 2096
+    },
+    {
+      "epoch": 2.6098319850653393,
+      "grad_norm": 3.456622362136841,
+      "learning_rate": 7.564575645756458e-07,
+      "loss": 1.3545,
+      "step": 2097
+    },
+    {
+      "epoch": 2.611076540136901,
+      "grad_norm": 3.488546133041382,
+      "learning_rate": 7.541512915129152e-07,
+      "loss": 1.3405,
+      "step": 2098
+    },
+    {
+      "epoch": 2.612321095208463,
+      "grad_norm": 3.4436495304107666,
+      "learning_rate": 7.518450184501845e-07,
+      "loss": 1.3242,
+      "step": 2099
+    },
+    {
+      "epoch": 2.613565650280025,
+      "grad_norm": 3.382150411605835,
+      "learning_rate": 7.49538745387454e-07,
+      "loss": 1.301,
+      "step": 2100
+    },
+    {
+      "epoch": 2.613565650280025,
+      "eval_loss": 1.6055735349655151,
+      "eval_runtime": 48.001,
+      "eval_samples_per_second": 20.833,
+      "eval_steps_per_second": 0.875,
+      "step": 2100
+    },
+    {
+      "epoch": 2.6148102053515867,
+      "grad_norm": 3.478010892868042,
+      "learning_rate": 7.472324723247233e-07,
+      "loss": 1.3565,
+      "step": 2101
+    },
+    {
+      "epoch": 2.6160547604231486,
+      "grad_norm": 3.4550209045410156,
+      "learning_rate": 7.449261992619927e-07,
+      "loss": 1.3732,
+      "step": 2102
+    },
+    {
+      "epoch": 2.6172993154947104,
+      "grad_norm": 3.3357841968536377,
+      "learning_rate": 7.42619926199262e-07,
+      "loss": 1.3072,
+      "step": 2103
+    },
+    {
+      "epoch": 2.6185438705662727,
+      "grad_norm": 3.4329988956451416,
+      "learning_rate": 7.403136531365313e-07,
+      "loss": 1.3353,
+      "step": 2104
+    },
+    {
+      "epoch": 2.6197884256378345,
+      "grad_norm": 3.390272378921509,
+      "learning_rate": 7.380073800738008e-07,
+      "loss": 1.3537,
+      "step": 2105
+    },
+    {
+      "epoch": 2.6210329807093964,
+      "grad_norm": 3.4377708435058594,
+      "learning_rate": 7.357011070110702e-07,
+      "loss": 1.357,
+      "step": 2106
+    },
+    {
+      "epoch": 2.6222775357809582,
+      "grad_norm": 3.383354425430298,
+      "learning_rate": 7.333948339483396e-07,
+      "loss": 1.3663,
+      "step": 2107
+    },
+    {
+      "epoch": 2.62352209085252,
+      "grad_norm": 3.364445209503174,
+      "learning_rate": 7.310885608856088e-07,
+      "loss": 1.3515,
+      "step": 2108
+    },
+    {
+      "epoch": 2.6247666459240824,
+      "grad_norm": 3.357224702835083,
+      "learning_rate": 7.287822878228782e-07,
+      "loss": 1.3436,
+      "step": 2109
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "grad_norm": 3.340733289718628,
+      "learning_rate": 7.264760147601477e-07,
+      "loss": 1.3481,
+      "step": 2110
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "eval_loss": 1.609564185142517,
+      "eval_runtime": 43.5251,
+      "eval_samples_per_second": 22.975,
+      "eval_steps_per_second": 0.965,
+      "step": 2110
+    },
+    {
+      "epoch": 2.627255756067206,
+      "grad_norm": 3.307832717895508,
+      "learning_rate": 7.241697416974171e-07,
+      "loss": 1.322,
+      "step": 2111
+    },
+    {
+      "epoch": 2.628500311138768,
+      "grad_norm": 3.3134219646453857,
+      "learning_rate": 7.218634686346863e-07,
+      "loss": 1.2888,
+      "step": 2112
+    },
+    {
+      "epoch": 2.6297448662103298,
+      "grad_norm": 3.5507848262786865,
+      "learning_rate": 7.195571955719557e-07,
+      "loss": 1.3986,
+      "step": 2113
+    },
+    {
+      "epoch": 2.6309894212818916,
+      "grad_norm": 3.26234769821167,
+      "learning_rate": 7.172509225092251e-07,
+      "loss": 1.3037,
+      "step": 2114
+    },
+    {
+      "epoch": 2.6322339763534535,
+      "grad_norm": 3.4239728450775146,
+      "learning_rate": 7.149446494464946e-07,
+      "loss": 1.326,
+      "step": 2115
+    },
+    {
+      "epoch": 2.6334785314250153,
+      "grad_norm": 3.470409870147705,
+      "learning_rate": 7.12638376383764e-07,
+      "loss": 1.3435,
+      "step": 2116
+    },
+    {
+      "epoch": 2.6347230864965776,
+      "grad_norm": 3.4207205772399902,
+      "learning_rate": 7.103321033210332e-07,
+      "loss": 1.3519,
+      "step": 2117
+    },
+    {
+      "epoch": 2.6359676415681395,
+      "grad_norm": 3.3702151775360107,
+      "learning_rate": 7.080258302583026e-07,
+      "loss": 1.3081,
+      "step": 2118
+    },
+    {
+      "epoch": 2.6372121966397013,
+      "grad_norm": 3.406705379486084,
+      "learning_rate": 7.05719557195572e-07,
+      "loss": 1.3698,
+      "step": 2119
+    },
+    {
+      "epoch": 2.638456751711263,
+      "grad_norm": 3.5395123958587646,
+      "learning_rate": 7.034132841328415e-07,
+      "loss": 1.3543,
+      "step": 2120
+    },
+    {
+      "epoch": 2.638456751711263,
+      "eval_loss": 1.6064285039901733,
+      "eval_runtime": 47.1124,
+      "eval_samples_per_second": 21.226,
+      "eval_steps_per_second": 0.891,
+      "step": 2120
+    },
+    {
+      "epoch": 2.639701306782825,
+      "grad_norm": 3.3317949771881104,
+      "learning_rate": 7.011070110701107e-07,
+      "loss": 1.3401,
+      "step": 2121
+    },
+    {
+      "epoch": 2.6409458618543873,
+      "grad_norm": 3.3046767711639404,
+      "learning_rate": 6.988007380073801e-07,
+      "loss": 1.355,
+      "step": 2122
+    },
+    {
+      "epoch": 2.642190416925949,
+      "grad_norm": 3.419886350631714,
+      "learning_rate": 6.964944649446495e-07,
+      "loss": 1.2955,
+      "step": 2123
+    },
+    {
+      "epoch": 2.643434971997511,
+      "grad_norm": 3.3755574226379395,
+      "learning_rate": 6.94188191881919e-07,
+      "loss": 1.3093,
+      "step": 2124
+    },
+    {
+      "epoch": 2.644679527069073,
+      "grad_norm": 3.490931510925293,
+      "learning_rate": 6.918819188191883e-07,
+      "loss": 1.3994,
+      "step": 2125
+    },
+    {
+      "epoch": 2.6459240821406347,
+      "grad_norm": 3.48911452293396,
+      "learning_rate": 6.895756457564576e-07,
+      "loss": 1.3449,
+      "step": 2126
+    },
+    {
+      "epoch": 2.6471686372121965,
+      "grad_norm": 3.5786304473876953,
+      "learning_rate": 6.87269372693727e-07,
+      "loss": 1.3569,
+      "step": 2127
+    },
+    {
+      "epoch": 2.6484131922837584,
+      "grad_norm": 3.500648260116577,
+      "learning_rate": 6.849630996309964e-07,
+      "loss": 1.349,
+      "step": 2128
+    },
+    {
+      "epoch": 2.6496577473553202,
+      "grad_norm": 3.644001007080078,
+      "learning_rate": 6.826568265682657e-07,
+      "loss": 1.3534,
+      "step": 2129
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "grad_norm": 3.3233609199523926,
+      "learning_rate": 6.803505535055351e-07,
+      "loss": 1.303,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "eval_loss": 1.6085610389709473,
+      "eval_runtime": 51.443,
+      "eval_samples_per_second": 19.439,
+      "eval_steps_per_second": 0.816,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6521468574984444,
+      "grad_norm": 3.413889169692993,
+      "learning_rate": 6.780442804428045e-07,
+      "loss": 1.335,
+      "step": 2131
+    },
+    {
+      "epoch": 2.653391412570006,
+      "grad_norm": 3.417112112045288,
+      "learning_rate": 6.757380073800739e-07,
+      "loss": 1.3768,
+      "step": 2132
+    },
+    {
+      "epoch": 2.654635967641568,
+      "grad_norm": 3.4491703510284424,
+      "learning_rate": 6.734317343173432e-07,
+      "loss": 1.3493,
+      "step": 2133
+    },
+    {
+      "epoch": 2.65588052271313,
+      "grad_norm": 3.3965368270874023,
+      "learning_rate": 6.711254612546127e-07,
+      "loss": 1.3328,
+      "step": 2134
+    },
+    {
+      "epoch": 2.657125077784692,
+      "grad_norm": 3.426530361175537,
+      "learning_rate": 6.68819188191882e-07,
+      "loss": 1.3548,
+      "step": 2135
+    },
+    {
+      "epoch": 2.658369632856254,
+      "grad_norm": 3.386624813079834,
+      "learning_rate": 6.665129151291514e-07,
+      "loss": 1.3523,
+      "step": 2136
+    },
+    {
+      "epoch": 2.659614187927816,
+      "grad_norm": 3.383504867553711,
+      "learning_rate": 6.642066420664207e-07,
+      "loss": 1.3356,
+      "step": 2137
+    },
+    {
+      "epoch": 2.6608587429993777,
+      "grad_norm": 3.3986196517944336,
+      "learning_rate": 6.619003690036901e-07,
+      "loss": 1.3605,
+      "step": 2138
+    },
+    {
+      "epoch": 2.6621032980709396,
+      "grad_norm": 3.4832820892333984,
+      "learning_rate": 6.595940959409594e-07,
+      "loss": 1.395,
+      "step": 2139
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "grad_norm": 3.498077154159546,
+      "learning_rate": 6.572878228782288e-07,
+      "loss": 1.347,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "eval_loss": 1.6076886653900146,
+      "eval_runtime": 47.7587,
+      "eval_samples_per_second": 20.939,
+      "eval_steps_per_second": 0.879,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6645924082140633,
+      "grad_norm": 3.6382765769958496,
+      "learning_rate": 6.549815498154982e-07,
+      "loss": 1.4288,
+      "step": 2141
+    },
+    {
+      "epoch": 2.665836963285625,
+      "grad_norm": 3.434046983718872,
+      "learning_rate": 6.526752767527676e-07,
+      "loss": 1.3258,
+      "step": 2142
+    },
+    {
+      "epoch": 2.6670815183571874,
+      "grad_norm": 3.381760358810425,
+      "learning_rate": 6.503690036900369e-07,
+      "loss": 1.3567,
+      "step": 2143
+    },
+    {
+      "epoch": 2.6683260734287493,
+      "grad_norm": 3.450314521789551,
+      "learning_rate": 6.480627306273062e-07,
+      "loss": 1.3504,
+      "step": 2144
+    },
+    {
+      "epoch": 2.669570628500311,
+      "grad_norm": 3.337059736251831,
+      "learning_rate": 6.457564575645757e-07,
+      "loss": 1.3664,
+      "step": 2145
+    },
+    {
+      "epoch": 2.670815183571873,
+      "grad_norm": 3.415597438812256,
+      "learning_rate": 6.434501845018451e-07,
+      "loss": 1.369,
+      "step": 2146
+    },
+    {
+      "epoch": 2.672059738643435,
+      "grad_norm": 3.331700325012207,
+      "learning_rate": 6.411439114391145e-07,
+      "loss": 1.3056,
+      "step": 2147
+    },
+    {
+      "epoch": 2.673304293714997,
+      "grad_norm": 3.439768075942993,
+      "learning_rate": 6.388376383763837e-07,
+      "loss": 1.3591,
+      "step": 2148
+    },
+    {
+      "epoch": 2.674548848786559,
+      "grad_norm": 3.4256532192230225,
+      "learning_rate": 6.365313653136531e-07,
+      "loss": 1.3581,
+      "step": 2149
+    },
+    {
+      "epoch": 2.675793403858121,
+      "grad_norm": 3.337472438812256,
+      "learning_rate": 6.342250922509226e-07,
+      "loss": 1.358,
+      "step": 2150
+    },
+    {
+      "epoch": 2.675793403858121,
+      "eval_loss": 1.6052860021591187,
+      "eval_runtime": 51.6432,
+      "eval_samples_per_second": 19.364,
+      "eval_steps_per_second": 0.813,
+      "step": 2150
+    },
+    {
+      "epoch": 2.6770379589296827,
+      "grad_norm": 3.458285331726074,
+      "learning_rate": 6.31918819188192e-07,
+      "loss": 1.346,
+      "step": 2151
+    },
+    {
+      "epoch": 2.6782825140012445,
+      "grad_norm": 3.3939685821533203,
+      "learning_rate": 6.296125461254612e-07,
+      "loss": 1.3634,
+      "step": 2152
+    },
+    {
+      "epoch": 2.6795270690728064,
+      "grad_norm": 3.4758291244506836,
+      "learning_rate": 6.273062730627306e-07,
+      "loss": 1.3913,
+      "step": 2153
+    },
+    {
+      "epoch": 2.680771624144368,
+      "grad_norm": 3.316906690597534,
+      "learning_rate": 6.25e-07,
+      "loss": 1.2944,
+      "step": 2154
+    },
+    {
+      "epoch": 2.68201617921593,
+      "grad_norm": 3.357058048248291,
+      "learning_rate": 6.226937269372695e-07,
+      "loss": 1.321,
+      "step": 2155
+    },
+    {
+      "epoch": 2.6832607342874923,
+      "grad_norm": 3.4547276496887207,
+      "learning_rate": 6.203874538745388e-07,
+      "loss": 1.3407,
+      "step": 2156
+    },
+    {
+      "epoch": 2.684505289359054,
+      "grad_norm": 3.3258347511291504,
+      "learning_rate": 6.180811808118082e-07,
+      "loss": 1.3526,
+      "step": 2157
+    },
+    {
+      "epoch": 2.685749844430616,
+      "grad_norm": 3.418070077896118,
+      "learning_rate": 6.157749077490775e-07,
+      "loss": 1.3764,
+      "step": 2158
+    },
+    {
+      "epoch": 2.686994399502178,
+      "grad_norm": 3.2903330326080322,
+      "learning_rate": 6.13468634686347e-07,
+      "loss": 1.3144,
+      "step": 2159
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "grad_norm": 3.340200901031494,
+      "learning_rate": 6.111623616236163e-07,
+      "loss": 1.3399,
+      "step": 2160
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "eval_loss": 1.6079058647155762,
+      "eval_runtime": 52.2924,
+      "eval_samples_per_second": 19.123,
+      "eval_steps_per_second": 0.803,
+      "step": 2160
+    },
+    {
+      "epoch": 2.689483509645302,
+      "grad_norm": 3.4739818572998047,
+      "learning_rate": 6.088560885608857e-07,
+      "loss": 1.333,
+      "step": 2161
+    },
+    {
+      "epoch": 2.690728064716864,
+      "grad_norm": 3.5063717365264893,
+      "learning_rate": 6.06549815498155e-07,
+      "loss": 1.3089,
+      "step": 2162
+    },
+    {
+      "epoch": 2.6919726197884257,
+      "grad_norm": 3.34189510345459,
+      "learning_rate": 6.042435424354244e-07,
+      "loss": 1.3173,
+      "step": 2163
+    },
+    {
+      "epoch": 2.6932171748599876,
+      "grad_norm": 3.385422706604004,
+      "learning_rate": 6.019372693726938e-07,
+      "loss": 1.3605,
+      "step": 2164
+    },
+    {
+      "epoch": 2.6944617299315494,
+      "grad_norm": 3.3808515071868896,
+      "learning_rate": 5.996309963099632e-07,
+      "loss": 1.3341,
+      "step": 2165
+    },
+    {
+      "epoch": 2.6957062850031113,
+      "grad_norm": 3.546069383621216,
+      "learning_rate": 5.973247232472326e-07,
+      "loss": 1.3706,
+      "step": 2166
+    },
+    {
+      "epoch": 2.696950840074673,
+      "grad_norm": 3.3705339431762695,
+      "learning_rate": 5.950184501845019e-07,
+      "loss": 1.3226,
+      "step": 2167
+    },
+    {
+      "epoch": 2.698195395146235,
+      "grad_norm": 3.4087791442871094,
+      "learning_rate": 5.927121771217713e-07,
+      "loss": 1.3285,
+      "step": 2168
+    },
+    {
+      "epoch": 2.6994399502177973,
+      "grad_norm": 3.2774477005004883,
+      "learning_rate": 5.904059040590407e-07,
+      "loss": 1.3128,
+      "step": 2169
+    },
+    {
+      "epoch": 2.700684505289359,
+      "grad_norm": 3.3653030395507812,
+      "learning_rate": 5.880996309963101e-07,
+      "loss": 1.3395,
+      "step": 2170
+    },
+    {
+      "epoch": 2.700684505289359,
+      "eval_loss": 1.608220100402832,
+      "eval_runtime": 53.3978,
+      "eval_samples_per_second": 18.727,
+      "eval_steps_per_second": 0.787,
+      "step": 2170
+    },
+    {
+      "epoch": 2.701929060360921,
+      "grad_norm": 3.394524335861206,
+      "learning_rate": 5.857933579335794e-07,
+      "loss": 1.2853,
+      "step": 2171
+    },
+    {
+      "epoch": 2.703173615432483,
+      "grad_norm": 3.3898823261260986,
+      "learning_rate": 5.834870848708487e-07,
+      "loss": 1.3494,
+      "step": 2172
+    },
+    {
+      "epoch": 2.7044181705040446,
+      "grad_norm": 3.394688367843628,
+      "learning_rate": 5.811808118081181e-07,
+      "loss": 1.3129,
+      "step": 2173
+    },
+    {
+      "epoch": 2.705662725575607,
+      "grad_norm": 3.4164483547210693,
+      "learning_rate": 5.788745387453875e-07,
+      "loss": 1.368,
+      "step": 2174
+    },
+    {
+      "epoch": 2.706907280647169,
+      "grad_norm": 3.2976245880126953,
+      "learning_rate": 5.765682656826569e-07,
+      "loss": 1.361,
+      "step": 2175
+    },
+    {
+      "epoch": 2.7081518357187306,
+      "grad_norm": 3.307027816772461,
+      "learning_rate": 5.742619926199262e-07,
+      "loss": 1.3509,
+      "step": 2176
+    },
+    {
+      "epoch": 2.7093963907902925,
+      "grad_norm": 3.4393889904022217,
+      "learning_rate": 5.719557195571956e-07,
+      "loss": 1.331,
+      "step": 2177
+    },
+    {
+      "epoch": 2.7106409458618543,
+      "grad_norm": 3.312872886657715,
+      "learning_rate": 5.696494464944649e-07,
+      "loss": 1.3541,
+      "step": 2178
+    },
+    {
+      "epoch": 2.711885500933416,
+      "grad_norm": 3.4782755374908447,
+      "learning_rate": 5.673431734317344e-07,
+      "loss": 1.3078,
+      "step": 2179
+    },
+    {
+      "epoch": 2.713130056004978,
+      "grad_norm": 3.457902193069458,
+      "learning_rate": 5.650369003690037e-07,
+      "loss": 1.3712,
+      "step": 2180
+    },
+    {
+      "epoch": 2.713130056004978,
+      "eval_loss": 1.6061460971832275,
+      "eval_runtime": 50.4797,
+      "eval_samples_per_second": 19.81,
+      "eval_steps_per_second": 0.832,
+      "step": 2180
+    },
+    {
+      "epoch": 2.71437461107654,
+      "grad_norm": 3.508249282836914,
+      "learning_rate": 5.627306273062731e-07,
+      "loss": 1.3948,
+      "step": 2181
+    },
+    {
+      "epoch": 2.715619166148102,
+      "grad_norm": 3.3043899536132812,
+      "learning_rate": 5.604243542435424e-07,
+      "loss": 1.3382,
+      "step": 2182
+    },
+    {
+      "epoch": 2.716863721219664,
+      "grad_norm": 3.4448275566101074,
+      "learning_rate": 5.581180811808118e-07,
+      "loss": 1.3422,
+      "step": 2183
+    },
+    {
+      "epoch": 2.718108276291226,
+      "grad_norm": 3.3538706302642822,
+      "learning_rate": 5.558118081180812e-07,
+      "loss": 1.3054,
+      "step": 2184
+    },
+    {
+      "epoch": 2.7193528313627877,
+      "grad_norm": 3.4720618724823,
+      "learning_rate": 5.535055350553506e-07,
+      "loss": 1.3525,
+      "step": 2185
+    },
+    {
+      "epoch": 2.7205973864343496,
+      "grad_norm": 3.370326519012451,
+      "learning_rate": 5.5119926199262e-07,
+      "loss": 1.3756,
+      "step": 2186
+    },
+    {
+      "epoch": 2.721841941505912,
+      "grad_norm": 3.4985008239746094,
+      "learning_rate": 5.488929889298893e-07,
+      "loss": 1.3509,
+      "step": 2187
+    },
+    {
+      "epoch": 2.7230864965774737,
+      "grad_norm": 3.3014514446258545,
+      "learning_rate": 5.465867158671587e-07,
+      "loss": 1.2928,
+      "step": 2188
+    },
+    {
+      "epoch": 2.7243310516490356,
+      "grad_norm": 3.491454839706421,
+      "learning_rate": 5.44280442804428e-07,
+      "loss": 1.315,
+      "step": 2189
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "grad_norm": 3.3876538276672363,
+      "learning_rate": 5.419741697416975e-07,
+      "loss": 1.3344,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "eval_loss": 1.606774926185608,
+      "eval_runtime": 49.375,
+      "eval_samples_per_second": 20.253,
+      "eval_steps_per_second": 0.851,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7268201617921592,
+      "grad_norm": 3.2686572074890137,
+      "learning_rate": 5.396678966789668e-07,
+      "loss": 1.3334,
+      "step": 2191
+    },
+    {
+      "epoch": 2.728064716863721,
+      "grad_norm": 3.539928913116455,
+      "learning_rate": 5.373616236162362e-07,
+      "loss": 1.3676,
+      "step": 2192
+    },
+    {
+      "epoch": 2.729309271935283,
+      "grad_norm": 3.424800395965576,
+      "learning_rate": 5.350553505535055e-07,
+      "loss": 1.3207,
+      "step": 2193
+    },
+    {
+      "epoch": 2.730553827006845,
+      "grad_norm": 3.459237575531006,
+      "learning_rate": 5.32749077490775e-07,
+      "loss": 1.3459,
+      "step": 2194
+    },
+    {
+      "epoch": 2.731798382078407,
+      "grad_norm": 3.39109468460083,
+      "learning_rate": 5.304428044280444e-07,
+      "loss": 1.3408,
+      "step": 2195
+    },
+    {
+      "epoch": 2.733042937149969,
+      "grad_norm": 3.4368112087249756,
+      "learning_rate": 5.281365313653137e-07,
+      "loss": 1.2679,
+      "step": 2196
+    },
+    {
+      "epoch": 2.734287492221531,
+      "grad_norm": 3.3579020500183105,
+      "learning_rate": 5.258302583025831e-07,
+      "loss": 1.3531,
+      "step": 2197
+    },
+    {
+      "epoch": 2.7355320472930926,
+      "grad_norm": 3.2895281314849854,
+      "learning_rate": 5.235239852398524e-07,
+      "loss": 1.3127,
+      "step": 2198
+    },
+    {
+      "epoch": 2.7367766023646545,
+      "grad_norm": 3.546879529953003,
+      "learning_rate": 5.212177121771219e-07,
+      "loss": 1.3816,
+      "step": 2199
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "grad_norm": 3.4015355110168457,
+      "learning_rate": 5.189114391143912e-07,
+      "loss": 1.3026,
+      "step": 2200
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "eval_loss": 1.6100414991378784,
+      "eval_runtime": 44.238,
+      "eval_samples_per_second": 22.605,
+      "eval_steps_per_second": 0.949,
+      "step": 2200
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2409,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.858080700596224e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2200/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/latest
new file mode 100644
index 0000000000000000000000000000000000000000..78dc01ce4ce1498b753887dfb7af92409705a69b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/latest
@@ -0,0 +1 @@
+global_step2300
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..cf9f7cc0f22e38cdfea7c36a09f25ba146158728
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b93ade60213134037a6e2a03060d84a67c5041245aff2ad374a5c97f337ee4c2
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..22044561af392bd37b8a514ffc971741f1a025c2
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e17eca23ec632d21ab0115d8db1a9360104fa7b08d928954eee2eed9adf46b8a
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..24228040a3cbfaaf0d0e62b9e12dbef736514695
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6614221a631cfb519fe3c961fa724d0b1c62628b10e8afa55fbb30d5380c88e3
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26faa3073c1eb0fc2bc47348851da9349eacad02
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92e4e58a248bf0af7c3e00e00322cb1b7fb7e630ff6a437f6ca349ad76b9160a
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6
+size 17210734
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/tokenizer_config.json
@@ -0,0 +1,2086 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅂",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128258": {
+      "content": "🄿",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e02d5db4cb1e55e94a20f91c202f198d3a3839d
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/trainer_state.json
@@ -0,0 +1,17973 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.8624766645924082,
+  "eval_steps": 10,
+  "global_step": 2300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8368,
+      "step": 1
+    },
+    {
+      "epoch": 0.002489110143123833,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.86,
+      "step": 2
+    },
+    {
+      "epoch": 0.00373366521468575,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8265,
+      "step": 3
+    },
+    {
+      "epoch": 0.004978220286247666,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8271,
+      "step": 4
+    },
+    {
+      "epoch": 0.006222775357809583,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8768,
+      "step": 5
+    },
+    {
+      "epoch": 0.0074673304293715,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8402,
+      "step": 6
+    },
+    {
+      "epoch": 0.008711885500933417,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.785,
+      "step": 7
+    },
+    {
+      "epoch": 0.009956440572495333,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8466,
+      "step": 8
+    },
+    {
+      "epoch": 0.01120099564405725,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.7673,
+      "step": 9
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "eval_loss": 2.8535053730010986,
+      "eval_runtime": 42.446,
+      "eval_samples_per_second": 23.559,
+      "eval_steps_per_second": 0.989,
+      "step": 10
+    },
+    {
+      "epoch": 0.013690105787181083,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8592,
+      "step": 11
+    },
+    {
+      "epoch": 0.014934660858743,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.804,
+      "step": 12
+    },
+    {
+      "epoch": 0.016179215930304917,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8076,
+      "step": 13
+    },
+    {
+      "epoch": 0.017423771001866834,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8219,
+      "step": 14
+    },
+    {
+      "epoch": 0.018668326073428748,
+      "grad_norm": 5.105710506439209,
+      "learning_rate": 2.0746887966804982e-08,
+      "loss": 2.7995,
+      "step": 15
+    },
+    {
+      "epoch": 0.019912881144990666,
+      "grad_norm": 5.156588077545166,
+      "learning_rate": 4.1493775933609963e-08,
+      "loss": 2.8577,
+      "step": 16
+    },
+    {
+      "epoch": 0.021157436216552583,
+      "grad_norm": 5.709373950958252,
+      "learning_rate": 6.224066390041494e-08,
+      "loss": 2.8433,
+      "step": 17
+    },
+    {
+      "epoch": 0.0224019912881145,
+      "grad_norm": 5.64658784866333,
+      "learning_rate": 8.298755186721993e-08,
+      "loss": 2.8428,
+      "step": 18
+    },
+    {
+      "epoch": 0.023646546359676415,
+      "grad_norm": 5.541937351226807,
+      "learning_rate": 1.037344398340249e-07,
+      "loss": 2.7785,
+      "step": 19
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "grad_norm": 5.233374118804932,
+      "learning_rate": 1.2448132780082988e-07,
+      "loss": 2.8599,
+      "step": 20
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "eval_loss": 2.8521182537078857,
+      "eval_runtime": 43.5664,
+      "eval_samples_per_second": 22.953,
+      "eval_steps_per_second": 0.964,
+      "step": 20
+    },
+    {
+      "epoch": 0.02613565650280025,
+      "grad_norm": 5.407598972320557,
+      "learning_rate": 1.4522821576763488e-07,
+      "loss": 2.8081,
+      "step": 21
+    },
+    {
+      "epoch": 0.027380211574362167,
+      "grad_norm": 5.0483317375183105,
+      "learning_rate": 1.6597510373443985e-07,
+      "loss": 2.8489,
+      "step": 22
+    },
+    {
+      "epoch": 0.02862476664592408,
+      "grad_norm": 4.985588550567627,
+      "learning_rate": 1.8672199170124483e-07,
+      "loss": 2.8375,
+      "step": 23
+    },
+    {
+      "epoch": 0.029869321717486,
+      "grad_norm": 5.201700687408447,
+      "learning_rate": 2.074688796680498e-07,
+      "loss": 2.8355,
+      "step": 24
+    },
+    {
+      "epoch": 0.031113876789047916,
+      "grad_norm": 5.009235382080078,
+      "learning_rate": 2.2821576763485478e-07,
+      "loss": 2.7774,
+      "step": 25
+    },
+    {
+      "epoch": 0.03235843186060983,
+      "grad_norm": 4.509210109710693,
+      "learning_rate": 2.4896265560165975e-07,
+      "loss": 2.7992,
+      "step": 26
+    },
+    {
+      "epoch": 0.03360298693217175,
+      "grad_norm": 4.529262065887451,
+      "learning_rate": 2.6970954356846476e-07,
+      "loss": 2.836,
+      "step": 27
+    },
+    {
+      "epoch": 0.03484754200373367,
+      "grad_norm": 4.526787281036377,
+      "learning_rate": 2.9045643153526976e-07,
+      "loss": 2.8128,
+      "step": 28
+    },
+    {
+      "epoch": 0.03609209707529558,
+      "grad_norm": 4.016609191894531,
+      "learning_rate": 3.112033195020747e-07,
+      "loss": 2.7947,
+      "step": 29
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "grad_norm": 3.4784562587738037,
+      "learning_rate": 3.319502074688797e-07,
+      "loss": 2.7854,
+      "step": 30
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "eval_loss": 2.8020100593566895,
+      "eval_runtime": 44.8464,
+      "eval_samples_per_second": 22.298,
+      "eval_steps_per_second": 0.937,
+      "step": 30
+    },
+    {
+      "epoch": 0.038581207218419414,
+      "grad_norm": 3.4887945652008057,
+      "learning_rate": 3.5269709543568466e-07,
+      "loss": 2.7908,
+      "step": 31
+    },
+    {
+      "epoch": 0.03982576228998133,
+      "grad_norm": 3.4533374309539795,
+      "learning_rate": 3.7344398340248966e-07,
+      "loss": 2.7358,
+      "step": 32
+    },
+    {
+      "epoch": 0.04107031736154325,
+      "grad_norm": 3.270954132080078,
+      "learning_rate": 3.941908713692946e-07,
+      "loss": 2.8013,
+      "step": 33
+    },
+    {
+      "epoch": 0.042314872433105166,
+      "grad_norm": 3.61322283744812,
+      "learning_rate": 4.149377593360996e-07,
+      "loss": 2.7521,
+      "step": 34
+    },
+    {
+      "epoch": 0.043559427504667084,
+      "grad_norm": 3.158719062805176,
+      "learning_rate": 4.3568464730290456e-07,
+      "loss": 2.7573,
+      "step": 35
+    },
+    {
+      "epoch": 0.044803982576229,
+      "grad_norm": 2.888746500015259,
+      "learning_rate": 4.5643153526970956e-07,
+      "loss": 2.7795,
+      "step": 36
+    },
+    {
+      "epoch": 0.04604853764779091,
+      "grad_norm": 3.022629499435425,
+      "learning_rate": 4.771784232365145e-07,
+      "loss": 2.7891,
+      "step": 37
+    },
+    {
+      "epoch": 0.04729309271935283,
+      "grad_norm": 3.038097381591797,
+      "learning_rate": 4.979253112033195e-07,
+      "loss": 2.7376,
+      "step": 38
+    },
+    {
+      "epoch": 0.04853764779091475,
+      "grad_norm": 2.9992239475250244,
+      "learning_rate": 5.186721991701245e-07,
+      "loss": 2.7279,
+      "step": 39
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "grad_norm": 2.908842086791992,
+      "learning_rate": 5.394190871369295e-07,
+      "loss": 2.6582,
+      "step": 40
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "eval_loss": 2.725968837738037,
+      "eval_runtime": 44.9498,
+      "eval_samples_per_second": 22.247,
+      "eval_steps_per_second": 0.934,
+      "step": 40
+    },
+    {
+      "epoch": 0.05102675793403858,
+      "grad_norm": 2.8025174140930176,
+      "learning_rate": 5.601659751037345e-07,
+      "loss": 2.6809,
+      "step": 41
+    },
+    {
+      "epoch": 0.0522713130056005,
+      "grad_norm": 2.7555699348449707,
+      "learning_rate": 5.809128630705395e-07,
+      "loss": 2.6954,
+      "step": 42
+    },
+    {
+      "epoch": 0.053515868077162417,
+      "grad_norm": 2.4732933044433594,
+      "learning_rate": 6.016597510373444e-07,
+      "loss": 2.7307,
+      "step": 43
+    },
+    {
+      "epoch": 0.054760423148724334,
+      "grad_norm": 2.451366424560547,
+      "learning_rate": 6.224066390041494e-07,
+      "loss": 2.7064,
+      "step": 44
+    },
+    {
+      "epoch": 0.056004978220286245,
+      "grad_norm": 2.5768678188323975,
+      "learning_rate": 6.431535269709543e-07,
+      "loss": 2.6466,
+      "step": 45
+    },
+    {
+      "epoch": 0.05724953329184816,
+      "grad_norm": 2.579332113265991,
+      "learning_rate": 6.639004149377594e-07,
+      "loss": 2.6127,
+      "step": 46
+    },
+    {
+      "epoch": 0.05849408836341008,
+      "grad_norm": 2.231207847595215,
+      "learning_rate": 6.846473029045644e-07,
+      "loss": 2.6429,
+      "step": 47
+    },
+    {
+      "epoch": 0.059738643434972,
+      "grad_norm": 2.134375810623169,
+      "learning_rate": 7.053941908713693e-07,
+      "loss": 2.6423,
+      "step": 48
+    },
+    {
+      "epoch": 0.060983198506533914,
+      "grad_norm": 2.0485494136810303,
+      "learning_rate": 7.261410788381744e-07,
+      "loss": 2.581,
+      "step": 49
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "grad_norm": 2.02909517288208,
+      "learning_rate": 7.468879668049793e-07,
+      "loss": 2.6357,
+      "step": 50
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "eval_loss": 2.632822036743164,
+      "eval_runtime": 47.0104,
+      "eval_samples_per_second": 21.272,
+      "eval_steps_per_second": 0.893,
+      "step": 50
+    },
+    {
+      "epoch": 0.06347230864965775,
+      "grad_norm": 1.892500877380371,
+      "learning_rate": 7.676348547717843e-07,
+      "loss": 2.5945,
+      "step": 51
+    },
+    {
+      "epoch": 0.06471686372121967,
+      "grad_norm": 2.107921600341797,
+      "learning_rate": 7.883817427385892e-07,
+      "loss": 2.5867,
+      "step": 52
+    },
+    {
+      "epoch": 0.06596141879278158,
+      "grad_norm": 1.8287527561187744,
+      "learning_rate": 8.091286307053943e-07,
+      "loss": 2.5862,
+      "step": 53
+    },
+    {
+      "epoch": 0.0672059738643435,
+      "grad_norm": 1.7342065572738647,
+      "learning_rate": 8.298755186721992e-07,
+      "loss": 2.5735,
+      "step": 54
+    },
+    {
+      "epoch": 0.06845052893590542,
+      "grad_norm": 1.7905038595199585,
+      "learning_rate": 8.506224066390042e-07,
+      "loss": 2.572,
+      "step": 55
+    },
+    {
+      "epoch": 0.06969508400746734,
+      "grad_norm": 1.9715144634246826,
+      "learning_rate": 8.713692946058091e-07,
+      "loss": 2.5797,
+      "step": 56
+    },
+    {
+      "epoch": 0.07093963907902924,
+      "grad_norm": 1.788966417312622,
+      "learning_rate": 8.921161825726142e-07,
+      "loss": 2.5567,
+      "step": 57
+    },
+    {
+      "epoch": 0.07218419415059116,
+      "grad_norm": 1.8978915214538574,
+      "learning_rate": 9.128630705394191e-07,
+      "loss": 2.5776,
+      "step": 58
+    },
+    {
+      "epoch": 0.07342874922215308,
+      "grad_norm": 1.6401689052581787,
+      "learning_rate": 9.336099585062241e-07,
+      "loss": 2.5662,
+      "step": 59
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "grad_norm": 1.8200798034667969,
+      "learning_rate": 9.54356846473029e-07,
+      "loss": 2.5001,
+      "step": 60
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "eval_loss": 2.569110870361328,
+      "eval_runtime": 47.3292,
+      "eval_samples_per_second": 21.129,
+      "eval_steps_per_second": 0.887,
+      "step": 60
+    },
+    {
+      "epoch": 0.07591785936527691,
+      "grad_norm": 1.5468674898147583,
+      "learning_rate": 9.751037344398341e-07,
+      "loss": 2.5308,
+      "step": 61
+    },
+    {
+      "epoch": 0.07716241443683883,
+      "grad_norm": 1.723684310913086,
+      "learning_rate": 9.95850622406639e-07,
+      "loss": 2.4907,
+      "step": 62
+    },
+    {
+      "epoch": 0.07840696950840075,
+      "grad_norm": 1.6346293687820435,
+      "learning_rate": 1.0165975103734441e-06,
+      "loss": 2.514,
+      "step": 63
+    },
+    {
+      "epoch": 0.07965152457996266,
+      "grad_norm": 1.8902088403701782,
+      "learning_rate": 1.037344398340249e-06,
+      "loss": 2.4859,
+      "step": 64
+    },
+    {
+      "epoch": 0.08089607965152458,
+      "grad_norm": 2.092611312866211,
+      "learning_rate": 1.058091286307054e-06,
+      "loss": 2.4888,
+      "step": 65
+    },
+    {
+      "epoch": 0.0821406347230865,
+      "grad_norm": 1.758482575416565,
+      "learning_rate": 1.078838174273859e-06,
+      "loss": 2.4537,
+      "step": 66
+    },
+    {
+      "epoch": 0.08338518979464841,
+      "grad_norm": 1.8941714763641357,
+      "learning_rate": 1.099585062240664e-06,
+      "loss": 2.5416,
+      "step": 67
+    },
+    {
+      "epoch": 0.08462974486621033,
+      "grad_norm": 1.7030234336853027,
+      "learning_rate": 1.120331950207469e-06,
+      "loss": 2.4791,
+      "step": 68
+    },
+    {
+      "epoch": 0.08587429993777225,
+      "grad_norm": 1.7602269649505615,
+      "learning_rate": 1.141078838174274e-06,
+      "loss": 2.4316,
+      "step": 69
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "grad_norm": 1.7635431289672852,
+      "learning_rate": 1.161825726141079e-06,
+      "loss": 2.4674,
+      "step": 70
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "eval_loss": 2.4922080039978027,
+      "eval_runtime": 45.3028,
+      "eval_samples_per_second": 22.074,
+      "eval_steps_per_second": 0.927,
+      "step": 70
+    },
+    {
+      "epoch": 0.08836341008089608,
+      "grad_norm": 2.0913803577423096,
+      "learning_rate": 1.182572614107884e-06,
+      "loss": 2.4341,
+      "step": 71
+    },
+    {
+      "epoch": 0.089607965152458,
+      "grad_norm": 1.8292183876037598,
+      "learning_rate": 1.2033195020746888e-06,
+      "loss": 2.4291,
+      "step": 72
+    },
+    {
+      "epoch": 0.09085252022401992,
+      "grad_norm": 2.206770181655884,
+      "learning_rate": 1.224066390041494e-06,
+      "loss": 2.3764,
+      "step": 73
+    },
+    {
+      "epoch": 0.09209707529558182,
+      "grad_norm": 1.8263559341430664,
+      "learning_rate": 1.2448132780082988e-06,
+      "loss": 2.4082,
+      "step": 74
+    },
+    {
+      "epoch": 0.09334163036714374,
+      "grad_norm": 1.9405455589294434,
+      "learning_rate": 1.2655601659751037e-06,
+      "loss": 2.4673,
+      "step": 75
+    },
+    {
+      "epoch": 0.09458618543870566,
+      "grad_norm": 2.138108015060425,
+      "learning_rate": 1.2863070539419086e-06,
+      "loss": 2.3879,
+      "step": 76
+    },
+    {
+      "epoch": 0.09583074051026758,
+      "grad_norm": 1.9024745225906372,
+      "learning_rate": 1.307053941908714e-06,
+      "loss": 2.4344,
+      "step": 77
+    },
+    {
+      "epoch": 0.0970752955818295,
+      "grad_norm": 1.9136689901351929,
+      "learning_rate": 1.3278008298755188e-06,
+      "loss": 2.474,
+      "step": 78
+    },
+    {
+      "epoch": 0.09831985065339141,
+      "grad_norm": 2.4752795696258545,
+      "learning_rate": 1.3485477178423237e-06,
+      "loss": 2.3412,
+      "step": 79
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "grad_norm": 1.8720006942749023,
+      "learning_rate": 1.3692946058091288e-06,
+      "loss": 2.3438,
+      "step": 80
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "eval_loss": 2.3980512619018555,
+      "eval_runtime": 48.2788,
+      "eval_samples_per_second": 20.713,
+      "eval_steps_per_second": 0.87,
+      "step": 80
+    },
+    {
+      "epoch": 0.10080896079651525,
+      "grad_norm": 2.671691656112671,
+      "learning_rate": 1.3900414937759337e-06,
+      "loss": 2.3336,
+      "step": 81
+    },
+    {
+      "epoch": 0.10205351586807716,
+      "grad_norm": 2.2953391075134277,
+      "learning_rate": 1.4107883817427386e-06,
+      "loss": 2.377,
+      "step": 82
+    },
+    {
+      "epoch": 0.10329807093963908,
+      "grad_norm": 3.009018898010254,
+      "learning_rate": 1.4315352697095435e-06,
+      "loss": 2.2977,
+      "step": 83
+    },
+    {
+      "epoch": 0.104542626011201,
+      "grad_norm": 2.664454936981201,
+      "learning_rate": 1.4522821576763488e-06,
+      "loss": 2.3271,
+      "step": 84
+    },
+    {
+      "epoch": 0.10578718108276292,
+      "grad_norm": 3.017303705215454,
+      "learning_rate": 1.4730290456431537e-06,
+      "loss": 2.3251,
+      "step": 85
+    },
+    {
+      "epoch": 0.10703173615432483,
+      "grad_norm": 2.634716510772705,
+      "learning_rate": 1.4937759336099586e-06,
+      "loss": 2.332,
+      "step": 86
+    },
+    {
+      "epoch": 0.10827629122588675,
+      "grad_norm": 3.059644937515259,
+      "learning_rate": 1.5145228215767635e-06,
+      "loss": 2.3478,
+      "step": 87
+    },
+    {
+      "epoch": 0.10952084629744867,
+      "grad_norm": 2.6962637901306152,
+      "learning_rate": 1.5352697095435686e-06,
+      "loss": 2.2792,
+      "step": 88
+    },
+    {
+      "epoch": 0.11076540136901059,
+      "grad_norm": 3.419729709625244,
+      "learning_rate": 1.5560165975103735e-06,
+      "loss": 2.2571,
+      "step": 89
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "grad_norm": 2.740781545639038,
+      "learning_rate": 1.5767634854771784e-06,
+      "loss": 2.2875,
+      "step": 90
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "eval_loss": 2.30843186378479,
+      "eval_runtime": 49.9264,
+      "eval_samples_per_second": 20.029,
+      "eval_steps_per_second": 0.841,
+      "step": 90
+    },
+    {
+      "epoch": 0.1132545115121344,
+      "grad_norm": 2.5608789920806885,
+      "learning_rate": 1.5975103734439833e-06,
+      "loss": 2.2592,
+      "step": 91
+    },
+    {
+      "epoch": 0.11449906658369632,
+      "grad_norm": 2.687999963760376,
+      "learning_rate": 1.6182572614107886e-06,
+      "loss": 2.2546,
+      "step": 92
+    },
+    {
+      "epoch": 0.11574362165525824,
+      "grad_norm": 2.695909023284912,
+      "learning_rate": 1.6390041493775935e-06,
+      "loss": 2.2525,
+      "step": 93
+    },
+    {
+      "epoch": 0.11698817672682016,
+      "grad_norm": 2.818357467651367,
+      "learning_rate": 1.6597510373443984e-06,
+      "loss": 2.216,
+      "step": 94
+    },
+    {
+      "epoch": 0.11823273179838208,
+      "grad_norm": 2.884119987487793,
+      "learning_rate": 1.6804979253112035e-06,
+      "loss": 2.2321,
+      "step": 95
+    },
+    {
+      "epoch": 0.119477286869944,
+      "grad_norm": 2.52104115486145,
+      "learning_rate": 1.7012448132780084e-06,
+      "loss": 2.199,
+      "step": 96
+    },
+    {
+      "epoch": 0.12072184194150591,
+      "grad_norm": 2.420313596725464,
+      "learning_rate": 1.7219917012448133e-06,
+      "loss": 2.1862,
+      "step": 97
+    },
+    {
+      "epoch": 0.12196639701306783,
+      "grad_norm": 2.8047542572021484,
+      "learning_rate": 1.7427385892116182e-06,
+      "loss": 2.1793,
+      "step": 98
+    },
+    {
+      "epoch": 0.12321095208462975,
+      "grad_norm": 2.836482286453247,
+      "learning_rate": 1.7634854771784235e-06,
+      "loss": 2.2271,
+      "step": 99
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "grad_norm": 2.5282301902770996,
+      "learning_rate": 1.7842323651452284e-06,
+      "loss": 2.1768,
+      "step": 100
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "eval_loss": 2.2312686443328857,
+      "eval_runtime": 49.2382,
+      "eval_samples_per_second": 20.309,
+      "eval_steps_per_second": 0.853,
+      "step": 100
+    },
+    {
+      "epoch": 0.12570006222775357,
+      "grad_norm": 3.3407280445098877,
+      "learning_rate": 1.8049792531120333e-06,
+      "loss": 2.1666,
+      "step": 101
+    },
+    {
+      "epoch": 0.1269446172993155,
+      "grad_norm": 2.4754133224487305,
+      "learning_rate": 1.8257261410788382e-06,
+      "loss": 2.1768,
+      "step": 102
+    },
+    {
+      "epoch": 0.1281891723708774,
+      "grad_norm": 3.430889129638672,
+      "learning_rate": 1.8464730290456433e-06,
+      "loss": 2.1953,
+      "step": 103
+    },
+    {
+      "epoch": 0.12943372744243933,
+      "grad_norm": 2.835294246673584,
+      "learning_rate": 1.8672199170124482e-06,
+      "loss": 2.146,
+      "step": 104
+    },
+    {
+      "epoch": 0.13067828251400124,
+      "grad_norm": 3.1532323360443115,
+      "learning_rate": 1.8879668049792531e-06,
+      "loss": 2.1729,
+      "step": 105
+    },
+    {
+      "epoch": 0.13192283758556317,
+      "grad_norm": 3.2278342247009277,
+      "learning_rate": 1.908713692946058e-06,
+      "loss": 2.1336,
+      "step": 106
+    },
+    {
+      "epoch": 0.13316739265712507,
+      "grad_norm": 2.7892515659332275,
+      "learning_rate": 1.929460580912863e-06,
+      "loss": 2.141,
+      "step": 107
+    },
+    {
+      "epoch": 0.134411947728687,
+      "grad_norm": 3.0179977416992188,
+      "learning_rate": 1.9502074688796682e-06,
+      "loss": 2.1255,
+      "step": 108
+    },
+    {
+      "epoch": 0.1356565028002489,
+      "grad_norm": 2.977935552597046,
+      "learning_rate": 1.970954356846473e-06,
+      "loss": 2.0986,
+      "step": 109
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "grad_norm": 3.3042492866516113,
+      "learning_rate": 1.991701244813278e-06,
+      "loss": 2.0992,
+      "step": 110
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "eval_loss": 2.171299695968628,
+      "eval_runtime": 42.9511,
+      "eval_samples_per_second": 23.282,
+      "eval_steps_per_second": 0.978,
+      "step": 110
+    },
+    {
+      "epoch": 0.13814561294337274,
+      "grad_norm": 2.9750890731811523,
+      "learning_rate": 2.012448132780083e-06,
+      "loss": 2.1116,
+      "step": 111
+    },
+    {
+      "epoch": 0.13939016801493467,
+      "grad_norm": 2.8199286460876465,
+      "learning_rate": 2.0331950207468883e-06,
+      "loss": 2.0822,
+      "step": 112
+    },
+    {
+      "epoch": 0.14063472308649658,
+      "grad_norm": 2.930532217025757,
+      "learning_rate": 2.053941908713693e-06,
+      "loss": 2.161,
+      "step": 113
+    },
+    {
+      "epoch": 0.14187927815805848,
+      "grad_norm": 3.3400321006774902,
+      "learning_rate": 2.074688796680498e-06,
+      "loss": 2.0953,
+      "step": 114
+    },
+    {
+      "epoch": 0.1431238332296204,
+      "grad_norm": 2.716564655303955,
+      "learning_rate": 2.095435684647303e-06,
+      "loss": 2.0757,
+      "step": 115
+    },
+    {
+      "epoch": 0.14436838830118232,
+      "grad_norm": 2.7586236000061035,
+      "learning_rate": 2.116182572614108e-06,
+      "loss": 2.0979,
+      "step": 116
+    },
+    {
+      "epoch": 0.14561294337274425,
+      "grad_norm": 2.622126340866089,
+      "learning_rate": 2.136929460580913e-06,
+      "loss": 2.0748,
+      "step": 117
+    },
+    {
+      "epoch": 0.14685749844430615,
+      "grad_norm": 2.646477460861206,
+      "learning_rate": 2.157676348547718e-06,
+      "loss": 2.0605,
+      "step": 118
+    },
+    {
+      "epoch": 0.14810205351586808,
+      "grad_norm": 2.8882410526275635,
+      "learning_rate": 2.178423236514523e-06,
+      "loss": 2.0501,
+      "step": 119
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "grad_norm": 2.8302247524261475,
+      "learning_rate": 2.199170124481328e-06,
+      "loss": 2.0592,
+      "step": 120
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "eval_loss": 2.1291966438293457,
+      "eval_runtime": 47.5399,
+      "eval_samples_per_second": 21.035,
+      "eval_steps_per_second": 0.883,
+      "step": 120
+    },
+    {
+      "epoch": 0.15059116365899192,
+      "grad_norm": 3.26979660987854,
+      "learning_rate": 2.219917012448133e-06,
+      "loss": 2.0422,
+      "step": 121
+    },
+    {
+      "epoch": 0.15183571873055382,
+      "grad_norm": 2.7483913898468018,
+      "learning_rate": 2.240663900414938e-06,
+      "loss": 2.0872,
+      "step": 122
+    },
+    {
+      "epoch": 0.15308027380211575,
+      "grad_norm": 3.1252024173736572,
+      "learning_rate": 2.2614107883817427e-06,
+      "loss": 2.0684,
+      "step": 123
+    },
+    {
+      "epoch": 0.15432482887367766,
+      "grad_norm": 2.9175283908843994,
+      "learning_rate": 2.282157676348548e-06,
+      "loss": 2.0522,
+      "step": 124
+    },
+    {
+      "epoch": 0.1555693839452396,
+      "grad_norm": 2.7950754165649414,
+      "learning_rate": 2.302904564315353e-06,
+      "loss": 2.065,
+      "step": 125
+    },
+    {
+      "epoch": 0.1568139390168015,
+      "grad_norm": 2.84049654006958,
+      "learning_rate": 2.323651452282158e-06,
+      "loss": 2.0781,
+      "step": 126
+    },
+    {
+      "epoch": 0.15805849408836342,
+      "grad_norm": 2.7396647930145264,
+      "learning_rate": 2.3443983402489627e-06,
+      "loss": 2.1758,
+      "step": 127
+    },
+    {
+      "epoch": 0.15930304915992533,
+      "grad_norm": 3.0765926837921143,
+      "learning_rate": 2.365145228215768e-06,
+      "loss": 2.0172,
+      "step": 128
+    },
+    {
+      "epoch": 0.16054760423148726,
+      "grad_norm": 3.217189073562622,
+      "learning_rate": 2.385892116182573e-06,
+      "loss": 2.0286,
+      "step": 129
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "grad_norm": 3.141545295715332,
+      "learning_rate": 2.4066390041493776e-06,
+      "loss": 2.0583,
+      "step": 130
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "eval_loss": 2.0946149826049805,
+      "eval_runtime": 54.92,
+      "eval_samples_per_second": 18.208,
+      "eval_steps_per_second": 0.765,
+      "step": 130
+    },
+    {
+      "epoch": 0.16303671437461106,
+      "grad_norm": 2.850052833557129,
+      "learning_rate": 2.4273858921161828e-06,
+      "loss": 2.0746,
+      "step": 131
+    },
+    {
+      "epoch": 0.164281269446173,
+      "grad_norm": 3.28913950920105,
+      "learning_rate": 2.448132780082988e-06,
+      "loss": 2.0539,
+      "step": 132
+    },
+    {
+      "epoch": 0.1655258245177349,
+      "grad_norm": 2.8819124698638916,
+      "learning_rate": 2.468879668049793e-06,
+      "loss": 1.9843,
+      "step": 133
+    },
+    {
+      "epoch": 0.16677037958929683,
+      "grad_norm": 3.6254632472991943,
+      "learning_rate": 2.4896265560165977e-06,
+      "loss": 2.0233,
+      "step": 134
+    },
+    {
+      "epoch": 0.16801493466085873,
+      "grad_norm": 2.7385146617889404,
+      "learning_rate": 2.5103734439834028e-06,
+      "loss": 1.9667,
+      "step": 135
+    },
+    {
+      "epoch": 0.16925948973242066,
+      "grad_norm": 2.9722647666931152,
+      "learning_rate": 2.5311203319502074e-06,
+      "loss": 1.9527,
+      "step": 136
+    },
+    {
+      "epoch": 0.17050404480398257,
+      "grad_norm": 3.139526605606079,
+      "learning_rate": 2.5518672199170125e-06,
+      "loss": 1.9967,
+      "step": 137
+    },
+    {
+      "epoch": 0.1717485998755445,
+      "grad_norm": 3.399920701980591,
+      "learning_rate": 2.5726141078838172e-06,
+      "loss": 1.9798,
+      "step": 138
+    },
+    {
+      "epoch": 0.1729931549471064,
+      "grad_norm": 2.8017327785491943,
+      "learning_rate": 2.5933609958506228e-06,
+      "loss": 1.983,
+      "step": 139
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "grad_norm": 3.9967198371887207,
+      "learning_rate": 2.614107883817428e-06,
+      "loss": 1.977,
+      "step": 140
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "eval_loss": 2.059037685394287,
+      "eval_runtime": 52.6232,
+      "eval_samples_per_second": 19.003,
+      "eval_steps_per_second": 0.798,
+      "step": 140
+    },
+    {
+      "epoch": 0.17548226509023024,
+      "grad_norm": 2.893092393875122,
+      "learning_rate": 2.6348547717842326e-06,
+      "loss": 1.9888,
+      "step": 141
+    },
+    {
+      "epoch": 0.17672682016179217,
+      "grad_norm": 3.634352207183838,
+      "learning_rate": 2.6556016597510377e-06,
+      "loss": 2.0726,
+      "step": 142
+    },
+    {
+      "epoch": 0.17797137523335407,
+      "grad_norm": 3.3651444911956787,
+      "learning_rate": 2.6763485477178423e-06,
+      "loss": 1.973,
+      "step": 143
+    },
+    {
+      "epoch": 0.179215930304916,
+      "grad_norm": 3.968986988067627,
+      "learning_rate": 2.6970954356846475e-06,
+      "loss": 1.9948,
+      "step": 144
+    },
+    {
+      "epoch": 0.1804604853764779,
+      "grad_norm": 3.20105242729187,
+      "learning_rate": 2.717842323651452e-06,
+      "loss": 1.9238,
+      "step": 145
+    },
+    {
+      "epoch": 0.18170504044803984,
+      "grad_norm": 3.648339033126831,
+      "learning_rate": 2.7385892116182577e-06,
+      "loss": 1.9923,
+      "step": 146
+    },
+    {
+      "epoch": 0.18294959551960174,
+      "grad_norm": 3.0856316089630127,
+      "learning_rate": 2.7593360995850628e-06,
+      "loss": 1.9749,
+      "step": 147
+    },
+    {
+      "epoch": 0.18419415059116365,
+      "grad_norm": 3.2681071758270264,
+      "learning_rate": 2.7800829875518675e-06,
+      "loss": 1.9304,
+      "step": 148
+    },
+    {
+      "epoch": 0.18543870566272558,
+      "grad_norm": 2.634958267211914,
+      "learning_rate": 2.8008298755186726e-06,
+      "loss": 1.9237,
+      "step": 149
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "grad_norm": 2.769491672515869,
+      "learning_rate": 2.8215767634854773e-06,
+      "loss": 1.8963,
+      "step": 150
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "eval_loss": 2.0394654273986816,
+      "eval_runtime": 55.2136,
+      "eval_samples_per_second": 18.111,
+      "eval_steps_per_second": 0.761,
+      "step": 150
+    },
+    {
+      "epoch": 0.1879278158058494,
+      "grad_norm": 3.4345781803131104,
+      "learning_rate": 2.8423236514522824e-06,
+      "loss": 1.9836,
+      "step": 151
+    },
+    {
+      "epoch": 0.18917237087741132,
+      "grad_norm": 3.115727424621582,
+      "learning_rate": 2.863070539419087e-06,
+      "loss": 1.9352,
+      "step": 152
+    },
+    {
+      "epoch": 0.19041692594897325,
+      "grad_norm": 3.050652503967285,
+      "learning_rate": 2.883817427385892e-06,
+      "loss": 1.9031,
+      "step": 153
+    },
+    {
+      "epoch": 0.19166148102053515,
+      "grad_norm": 2.9404428005218506,
+      "learning_rate": 2.9045643153526977e-06,
+      "loss": 1.9196,
+      "step": 154
+    },
+    {
+      "epoch": 0.19290603609209708,
+      "grad_norm": 3.080810785293579,
+      "learning_rate": 2.9253112033195024e-06,
+      "loss": 1.9405,
+      "step": 155
+    },
+    {
+      "epoch": 0.194150591163659,
+      "grad_norm": 3.058558702468872,
+      "learning_rate": 2.9460580912863075e-06,
+      "loss": 1.9052,
+      "step": 156
+    },
+    {
+      "epoch": 0.19539514623522092,
+      "grad_norm": 3.307955026626587,
+      "learning_rate": 2.966804979253112e-06,
+      "loss": 1.9245,
+      "step": 157
+    },
+    {
+      "epoch": 0.19663970130678282,
+      "grad_norm": 2.845506191253662,
+      "learning_rate": 2.9875518672199173e-06,
+      "loss": 1.9461,
+      "step": 158
+    },
+    {
+      "epoch": 0.19788425637834475,
+      "grad_norm": 2.758654832839966,
+      "learning_rate": 3.008298755186722e-06,
+      "loss": 1.8885,
+      "step": 159
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "grad_norm": 3.161252737045288,
+      "learning_rate": 3.029045643153527e-06,
+      "loss": 1.8535,
+      "step": 160
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "eval_loss": 2.018101453781128,
+      "eval_runtime": 51.0744,
+      "eval_samples_per_second": 19.579,
+      "eval_steps_per_second": 0.822,
+      "step": 160
+    },
+    {
+      "epoch": 0.2003733665214686,
+      "grad_norm": 3.196847677230835,
+      "learning_rate": 3.0497925311203326e-06,
+      "loss": 1.8853,
+      "step": 161
+    },
+    {
+      "epoch": 0.2016179215930305,
+      "grad_norm": 2.8364663124084473,
+      "learning_rate": 3.0705394190871373e-06,
+      "loss": 1.9463,
+      "step": 162
+    },
+    {
+      "epoch": 0.2028624766645924,
+      "grad_norm": 3.0874054431915283,
+      "learning_rate": 3.0912863070539424e-06,
+      "loss": 1.9254,
+      "step": 163
+    },
+    {
+      "epoch": 0.20410703173615433,
+      "grad_norm": 2.7914493083953857,
+      "learning_rate": 3.112033195020747e-06,
+      "loss": 1.9213,
+      "step": 164
+    },
+    {
+      "epoch": 0.20535158680771623,
+      "grad_norm": 3.3871428966522217,
+      "learning_rate": 3.132780082987552e-06,
+      "loss": 1.8993,
+      "step": 165
+    },
+    {
+      "epoch": 0.20659614187927816,
+      "grad_norm": 3.096653461456299,
+      "learning_rate": 3.153526970954357e-06,
+      "loss": 1.8827,
+      "step": 166
+    },
+    {
+      "epoch": 0.20784069695084006,
+      "grad_norm": 2.8276076316833496,
+      "learning_rate": 3.174273858921162e-06,
+      "loss": 1.912,
+      "step": 167
+    },
+    {
+      "epoch": 0.209085252022402,
+      "grad_norm": 3.3058435916900635,
+      "learning_rate": 3.1950207468879666e-06,
+      "loss": 1.8381,
+      "step": 168
+    },
+    {
+      "epoch": 0.2103298070939639,
+      "grad_norm": 3.5017333030700684,
+      "learning_rate": 3.215767634854772e-06,
+      "loss": 1.9544,
+      "step": 169
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "grad_norm": 3.457296133041382,
+      "learning_rate": 3.2365145228215773e-06,
+      "loss": 1.922,
+      "step": 170
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "eval_loss": 1.988455891609192,
+      "eval_runtime": 56.2099,
+      "eval_samples_per_second": 17.79,
+      "eval_steps_per_second": 0.747,
+      "step": 170
+    },
+    {
+      "epoch": 0.21281891723708773,
+      "grad_norm": 3.1964566707611084,
+      "learning_rate": 3.257261410788382e-06,
+      "loss": 1.8925,
+      "step": 171
+    },
+    {
+      "epoch": 0.21406347230864967,
+      "grad_norm": 3.234652042388916,
+      "learning_rate": 3.278008298755187e-06,
+      "loss": 1.884,
+      "step": 172
+    },
+    {
+      "epoch": 0.21530802738021157,
+      "grad_norm": 3.14414119720459,
+      "learning_rate": 3.2987551867219918e-06,
+      "loss": 1.8905,
+      "step": 173
+    },
+    {
+      "epoch": 0.2165525824517735,
+      "grad_norm": 3.606379508972168,
+      "learning_rate": 3.319502074688797e-06,
+      "loss": 1.946,
+      "step": 174
+    },
+    {
+      "epoch": 0.2177971375233354,
+      "grad_norm": 2.986646890640259,
+      "learning_rate": 3.3402489626556016e-06,
+      "loss": 1.8691,
+      "step": 175
+    },
+    {
+      "epoch": 0.21904169259489734,
+      "grad_norm": 3.7298269271850586,
+      "learning_rate": 3.360995850622407e-06,
+      "loss": 1.8717,
+      "step": 176
+    },
+    {
+      "epoch": 0.22028624766645924,
+      "grad_norm": 3.422295093536377,
+      "learning_rate": 3.381742738589212e-06,
+      "loss": 1.8568,
+      "step": 177
+    },
+    {
+      "epoch": 0.22153080273802117,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.8471,
+      "step": 178
+    },
+    {
+      "epoch": 0.22277535780958307,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.9303,
+      "step": 179
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "grad_norm": 3.506223440170288,
+      "learning_rate": 3.423236514522822e-06,
+      "loss": 1.8237,
+      "step": 180
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "eval_loss": 1.9734643697738647,
+      "eval_runtime": 52.2815,
+      "eval_samples_per_second": 19.127,
+      "eval_steps_per_second": 0.803,
+      "step": 180
+    },
+    {
+      "epoch": 0.2252644679527069,
+      "grad_norm": 3.0156619548797607,
+      "learning_rate": 3.4439834024896267e-06,
+      "loss": 1.908,
+      "step": 181
+    },
+    {
+      "epoch": 0.2265090230242688,
+      "grad_norm": 3.55517315864563,
+      "learning_rate": 3.4647302904564318e-06,
+      "loss": 1.9104,
+      "step": 182
+    },
+    {
+      "epoch": 0.22775357809583074,
+      "grad_norm": 3.144984006881714,
+      "learning_rate": 3.4854771784232365e-06,
+      "loss": 1.8198,
+      "step": 183
+    },
+    {
+      "epoch": 0.22899813316739265,
+      "grad_norm": 4.302074432373047,
+      "learning_rate": 3.5062240663900416e-06,
+      "loss": 1.8237,
+      "step": 184
+    },
+    {
+      "epoch": 0.23024268823895458,
+      "grad_norm": 3.0522446632385254,
+      "learning_rate": 3.526970954356847e-06,
+      "loss": 1.8409,
+      "step": 185
+    },
+    {
+      "epoch": 0.23148724331051648,
+      "grad_norm": 3.8607394695281982,
+      "learning_rate": 3.5477178423236518e-06,
+      "loss": 1.8807,
+      "step": 186
+    },
+    {
+      "epoch": 0.23273179838207841,
+      "grad_norm": 2.9236302375793457,
+      "learning_rate": 3.568464730290457e-06,
+      "loss": 1.7873,
+      "step": 187
+    },
+    {
+      "epoch": 0.23397635345364032,
+      "grad_norm": 4.013780117034912,
+      "learning_rate": 3.5892116182572616e-06,
+      "loss": 1.7909,
+      "step": 188
+    },
+    {
+      "epoch": 0.23522090852520225,
+      "grad_norm": 3.0933122634887695,
+      "learning_rate": 3.6099585062240667e-06,
+      "loss": 1.8469,
+      "step": 189
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "grad_norm": 3.487816095352173,
+      "learning_rate": 3.6307053941908714e-06,
+      "loss": 1.8469,
+      "step": 190
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "eval_loss": 1.9604240655899048,
+      "eval_runtime": 53.4604,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 0.786,
+      "step": 190
+    },
+    {
+      "epoch": 0.23771001866832608,
+      "grad_norm": 2.829219341278076,
+      "learning_rate": 3.6514522821576765e-06,
+      "loss": 1.8639,
+      "step": 191
+    },
+    {
+      "epoch": 0.238954573739888,
+      "grad_norm": 3.595534324645996,
+      "learning_rate": 3.672199170124482e-06,
+      "loss": 1.8988,
+      "step": 192
+    },
+    {
+      "epoch": 0.24019912881144992,
+      "grad_norm": 2.9022483825683594,
+      "learning_rate": 3.6929460580912867e-06,
+      "loss": 1.8972,
+      "step": 193
+    },
+    {
+      "epoch": 0.24144368388301182,
+      "grad_norm": 3.0667994022369385,
+      "learning_rate": 3.713692946058092e-06,
+      "loss": 1.8499,
+      "step": 194
+    },
+    {
+      "epoch": 0.24268823895457373,
+      "grad_norm": 3.0498485565185547,
+      "learning_rate": 3.7344398340248965e-06,
+      "loss": 1.8341,
+      "step": 195
+    },
+    {
+      "epoch": 0.24393279402613566,
+      "grad_norm": 4.058084964752197,
+      "learning_rate": 3.7551867219917016e-06,
+      "loss": 1.8641,
+      "step": 196
+    },
+    {
+      "epoch": 0.24517734909769756,
+      "grad_norm": 2.9995028972625732,
+      "learning_rate": 3.7759336099585063e-06,
+      "loss": 1.7885,
+      "step": 197
+    },
+    {
+      "epoch": 0.2464219041692595,
+      "grad_norm": 3.254704475402832,
+      "learning_rate": 3.7966804979253114e-06,
+      "loss": 1.7544,
+      "step": 198
+    },
+    {
+      "epoch": 0.2476664592408214,
+      "grad_norm": 2.9126744270324707,
+      "learning_rate": 3.817427385892116e-06,
+      "loss": 1.8461,
+      "step": 199
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "grad_norm": 3.475247621536255,
+      "learning_rate": 3.838174273858922e-06,
+      "loss": 1.8095,
+      "step": 200
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "eval_loss": 1.9373760223388672,
+      "eval_runtime": 51.8733,
+      "eval_samples_per_second": 19.278,
+      "eval_steps_per_second": 0.81,
+      "step": 200
+    },
+    {
+      "epoch": 0.25015556938394523,
+      "grad_norm": 3.2663731575012207,
+      "learning_rate": 3.858921161825726e-06,
+      "loss": 1.8076,
+      "step": 201
+    },
+    {
+      "epoch": 0.25140012445550713,
+      "grad_norm": 3.1151726245880127,
+      "learning_rate": 3.879668049792531e-06,
+      "loss": 1.8143,
+      "step": 202
+    },
+    {
+      "epoch": 0.2526446795270691,
+      "grad_norm": 3.102038860321045,
+      "learning_rate": 3.9004149377593365e-06,
+      "loss": 1.8279,
+      "step": 203
+    },
+    {
+      "epoch": 0.253889234598631,
+      "grad_norm": 3.370642900466919,
+      "learning_rate": 3.921161825726142e-06,
+      "loss": 1.8522,
+      "step": 204
+    },
+    {
+      "epoch": 0.2551337896701929,
+      "grad_norm": 3.116128921508789,
+      "learning_rate": 3.941908713692946e-06,
+      "loss": 1.8104,
+      "step": 205
+    },
+    {
+      "epoch": 0.2563783447417548,
+      "grad_norm": 3.12028431892395,
+      "learning_rate": 3.962655601659751e-06,
+      "loss": 1.7723,
+      "step": 206
+    },
+    {
+      "epoch": 0.25762289981331676,
+      "grad_norm": 3.1251418590545654,
+      "learning_rate": 3.983402489626556e-06,
+      "loss": 1.863,
+      "step": 207
+    },
+    {
+      "epoch": 0.25886745488487867,
+      "grad_norm": 3.073702812194824,
+      "learning_rate": 4.004149377593361e-06,
+      "loss": 1.805,
+      "step": 208
+    },
+    {
+      "epoch": 0.26011200995644057,
+      "grad_norm": 2.9772469997406006,
+      "learning_rate": 4.024896265560166e-06,
+      "loss": 1.8251,
+      "step": 209
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "grad_norm": 2.9970037937164307,
+      "learning_rate": 4.045643153526971e-06,
+      "loss": 1.8386,
+      "step": 210
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "eval_loss": 1.9167065620422363,
+      "eval_runtime": 42.3328,
+      "eval_samples_per_second": 23.622,
+      "eval_steps_per_second": 0.992,
+      "step": 210
+    },
+    {
+      "epoch": 0.26260112009956443,
+      "grad_norm": 3.1391561031341553,
+      "learning_rate": 4.0663900414937765e-06,
+      "loss": 1.8109,
+      "step": 211
+    },
+    {
+      "epoch": 0.26384567517112634,
+      "grad_norm": 3.161538600921631,
+      "learning_rate": 4.087136929460581e-06,
+      "loss": 1.8441,
+      "step": 212
+    },
+    {
+      "epoch": 0.26509023024268824,
+      "grad_norm": 3.12412166595459,
+      "learning_rate": 4.107883817427386e-06,
+      "loss": 1.7736,
+      "step": 213
+    },
+    {
+      "epoch": 0.26633478531425014,
+      "grad_norm": 3.3241145610809326,
+      "learning_rate": 4.128630705394191e-06,
+      "loss": 1.8067,
+      "step": 214
+    },
+    {
+      "epoch": 0.26757934038581205,
+      "grad_norm": 3.503307819366455,
+      "learning_rate": 4.149377593360996e-06,
+      "loss": 1.7858,
+      "step": 215
+    },
+    {
+      "epoch": 0.268823895457374,
+      "grad_norm": 3.1009578704833984,
+      "learning_rate": 4.170124481327801e-06,
+      "loss": 1.7542,
+      "step": 216
+    },
+    {
+      "epoch": 0.2700684505289359,
+      "grad_norm": 3.2236011028289795,
+      "learning_rate": 4.190871369294606e-06,
+      "loss": 1.8006,
+      "step": 217
+    },
+    {
+      "epoch": 0.2713130056004978,
+      "grad_norm": 3.048935651779175,
+      "learning_rate": 4.211618257261411e-06,
+      "loss": 1.7706,
+      "step": 218
+    },
+    {
+      "epoch": 0.2725575606720597,
+      "grad_norm": 3.3416147232055664,
+      "learning_rate": 4.232365145228216e-06,
+      "loss": 1.8108,
+      "step": 219
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "grad_norm": 3.2010738849639893,
+      "learning_rate": 4.253112033195021e-06,
+      "loss": 1.8561,
+      "step": 220
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "eval_loss": 1.9111930131912231,
+      "eval_runtime": 50.4077,
+      "eval_samples_per_second": 19.838,
+      "eval_steps_per_second": 0.833,
+      "step": 220
+    },
+    {
+      "epoch": 0.2750466708151836,
+      "grad_norm": 3.4033374786376953,
+      "learning_rate": 4.273858921161826e-06,
+      "loss": 1.8218,
+      "step": 221
+    },
+    {
+      "epoch": 0.2762912258867455,
+      "grad_norm": 3.2086002826690674,
+      "learning_rate": 4.294605809128631e-06,
+      "loss": 1.8213,
+      "step": 222
+    },
+    {
+      "epoch": 0.2775357809583074,
+      "grad_norm": 4.090956211090088,
+      "learning_rate": 4.315352697095436e-06,
+      "loss": 1.8455,
+      "step": 223
+    },
+    {
+      "epoch": 0.27878033602986935,
+      "grad_norm": 3.1142985820770264,
+      "learning_rate": 4.336099585062241e-06,
+      "loss": 1.7737,
+      "step": 224
+    },
+    {
+      "epoch": 0.28002489110143125,
+      "grad_norm": 3.369669198989868,
+      "learning_rate": 4.356846473029046e-06,
+      "loss": 1.824,
+      "step": 225
+    },
+    {
+      "epoch": 0.28126944617299315,
+      "grad_norm": 3.165672779083252,
+      "learning_rate": 4.3775933609958506e-06,
+      "loss": 1.7768,
+      "step": 226
+    },
+    {
+      "epoch": 0.28251400124455506,
+      "grad_norm": 3.3343470096588135,
+      "learning_rate": 4.398340248962656e-06,
+      "loss": 1.7278,
+      "step": 227
+    },
+    {
+      "epoch": 0.28375855631611696,
+      "grad_norm": 3.2574994564056396,
+      "learning_rate": 4.419087136929461e-06,
+      "loss": 1.8439,
+      "step": 228
+    },
+    {
+      "epoch": 0.2850031113876789,
+      "grad_norm": 3.043928623199463,
+      "learning_rate": 4.439834024896266e-06,
+      "loss": 1.818,
+      "step": 229
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "grad_norm": 2.9703125953674316,
+      "learning_rate": 4.460580912863071e-06,
+      "loss": 1.773,
+      "step": 230
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "eval_loss": 1.898109793663025,
+      "eval_runtime": 45.6023,
+      "eval_samples_per_second": 21.929,
+      "eval_steps_per_second": 0.921,
+      "step": 230
+    },
+    {
+      "epoch": 0.2874922215308027,
+      "grad_norm": 2.9997220039367676,
+      "learning_rate": 4.481327800829876e-06,
+      "loss": 1.7727,
+      "step": 231
+    },
+    {
+      "epoch": 0.28873677660236463,
+      "grad_norm": 3.1358482837677,
+      "learning_rate": 4.502074688796681e-06,
+      "loss": 1.7764,
+      "step": 232
+    },
+    {
+      "epoch": 0.2899813316739266,
+      "grad_norm": 3.395747661590576,
+      "learning_rate": 4.5228215767634855e-06,
+      "loss": 1.7826,
+      "step": 233
+    },
+    {
+      "epoch": 0.2912258867454885,
+      "grad_norm": 3.7494754791259766,
+      "learning_rate": 4.543568464730291e-06,
+      "loss": 1.7181,
+      "step": 234
+    },
+    {
+      "epoch": 0.2924704418170504,
+      "grad_norm": 3.42293381690979,
+      "learning_rate": 4.564315352697096e-06,
+      "loss": 1.7773,
+      "step": 235
+    },
+    {
+      "epoch": 0.2937149968886123,
+      "grad_norm": 3.2524514198303223,
+      "learning_rate": 4.585062240663901e-06,
+      "loss": 1.7294,
+      "step": 236
+    },
+    {
+      "epoch": 0.29495955196017426,
+      "grad_norm": 3.657869577407837,
+      "learning_rate": 4.605809128630706e-06,
+      "loss": 1.762,
+      "step": 237
+    },
+    {
+      "epoch": 0.29620410703173616,
+      "grad_norm": 3.127372980117798,
+      "learning_rate": 4.626556016597511e-06,
+      "loss": 1.7623,
+      "step": 238
+    },
+    {
+      "epoch": 0.29744866210329807,
+      "grad_norm": 4.166962146759033,
+      "learning_rate": 4.647302904564316e-06,
+      "loss": 1.6995,
+      "step": 239
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "grad_norm": 3.094264030456543,
+      "learning_rate": 4.66804979253112e-06,
+      "loss": 1.7469,
+      "step": 240
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "eval_loss": 1.8908178806304932,
+      "eval_runtime": 44.6332,
+      "eval_samples_per_second": 22.405,
+      "eval_steps_per_second": 0.941,
+      "step": 240
+    },
+    {
+      "epoch": 0.29993777224642193,
+      "grad_norm": 3.8644745349884033,
+      "learning_rate": 4.6887966804979255e-06,
+      "loss": 1.7644,
+      "step": 241
+    },
+    {
+      "epoch": 0.30118232731798383,
+      "grad_norm": 3.1488852500915527,
+      "learning_rate": 4.709543568464731e-06,
+      "loss": 1.772,
+      "step": 242
+    },
+    {
+      "epoch": 0.30242688238954574,
+      "grad_norm": 3.3179638385772705,
+      "learning_rate": 4.730290456431536e-06,
+      "loss": 1.7588,
+      "step": 243
+    },
+    {
+      "epoch": 0.30367143746110764,
+      "grad_norm": 3.328355550765991,
+      "learning_rate": 4.751037344398341e-06,
+      "loss": 1.7417,
+      "step": 244
+    },
+    {
+      "epoch": 0.30491599253266954,
+      "grad_norm": 3.2690482139587402,
+      "learning_rate": 4.771784232365146e-06,
+      "loss": 1.7562,
+      "step": 245
+    },
+    {
+      "epoch": 0.3061605476042315,
+      "grad_norm": 3.2759454250335693,
+      "learning_rate": 4.792531120331951e-06,
+      "loss": 1.7536,
+      "step": 246
+    },
+    {
+      "epoch": 0.3074051026757934,
+      "grad_norm": 2.937964916229248,
+      "learning_rate": 4.813278008298755e-06,
+      "loss": 1.7518,
+      "step": 247
+    },
+    {
+      "epoch": 0.3086496577473553,
+      "grad_norm": 3.0995302200317383,
+      "learning_rate": 4.83402489626556e-06,
+      "loss": 1.7593,
+      "step": 248
+    },
+    {
+      "epoch": 0.3098942128189172,
+      "grad_norm": 3.230459213256836,
+      "learning_rate": 4.8547717842323655e-06,
+      "loss": 1.6962,
+      "step": 249
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "grad_norm": 3.242576837539673,
+      "learning_rate": 4.875518672199171e-06,
+      "loss": 1.7307,
+      "step": 250
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "eval_loss": 1.8805371522903442,
+      "eval_runtime": 45.9886,
+      "eval_samples_per_second": 21.745,
+      "eval_steps_per_second": 0.913,
+      "step": 250
+    },
+    {
+      "epoch": 0.3123833229620411,
+      "grad_norm": 3.097045421600342,
+      "learning_rate": 4.896265560165976e-06,
+      "loss": 1.7438,
+      "step": 251
+    },
+    {
+      "epoch": 0.313627878033603,
+      "grad_norm": 3.2428948879241943,
+      "learning_rate": 4.91701244813278e-06,
+      "loss": 1.7179,
+      "step": 252
+    },
+    {
+      "epoch": 0.3148724331051649,
+      "grad_norm": 3.196274518966675,
+      "learning_rate": 4.937759336099586e-06,
+      "loss": 1.7855,
+      "step": 253
+    },
+    {
+      "epoch": 0.31611698817672684,
+      "grad_norm": 2.978203535079956,
+      "learning_rate": 4.95850622406639e-06,
+      "loss": 1.7144,
+      "step": 254
+    },
+    {
+      "epoch": 0.31736154324828875,
+      "grad_norm": 3.2641701698303223,
+      "learning_rate": 4.979253112033195e-06,
+      "loss": 1.7427,
+      "step": 255
+    },
+    {
+      "epoch": 0.31860609831985065,
+      "grad_norm": 2.7441232204437256,
+      "learning_rate": 5e-06,
+      "loss": 1.7619,
+      "step": 256
+    },
+    {
+      "epoch": 0.31985065339141255,
+      "grad_norm": 3.0723495483398438,
+      "learning_rate": 4.99769372693727e-06,
+      "loss": 1.8017,
+      "step": 257
+    },
+    {
+      "epoch": 0.3210952084629745,
+      "grad_norm": 3.104752540588379,
+      "learning_rate": 4.995387453874539e-06,
+      "loss": 1.7761,
+      "step": 258
+    },
+    {
+      "epoch": 0.3223397635345364,
+      "grad_norm": 3.138627052307129,
+      "learning_rate": 4.993081180811809e-06,
+      "loss": 1.7879,
+      "step": 259
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "grad_norm": 2.8567333221435547,
+      "learning_rate": 4.990774907749078e-06,
+      "loss": 1.7561,
+      "step": 260
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "eval_loss": 1.8606494665145874,
+      "eval_runtime": 46.453,
+      "eval_samples_per_second": 21.527,
+      "eval_steps_per_second": 0.904,
+      "step": 260
+    },
+    {
+      "epoch": 0.3248288736776602,
+      "grad_norm": 3.2696540355682373,
+      "learning_rate": 4.988468634686347e-06,
+      "loss": 1.7201,
+      "step": 261
+    },
+    {
+      "epoch": 0.3260734287492221,
+      "grad_norm": 2.6692731380462646,
+      "learning_rate": 4.986162361623617e-06,
+      "loss": 1.7264,
+      "step": 262
+    },
+    {
+      "epoch": 0.3273179838207841,
+      "grad_norm": 3.047549247741699,
+      "learning_rate": 4.983856088560886e-06,
+      "loss": 1.7362,
+      "step": 263
+    },
+    {
+      "epoch": 0.328562538892346,
+      "grad_norm": 3.0667457580566406,
+      "learning_rate": 4.981549815498156e-06,
+      "loss": 1.7321,
+      "step": 264
+    },
+    {
+      "epoch": 0.3298070939639079,
+      "grad_norm": 2.560047149658203,
+      "learning_rate": 4.979243542435424e-06,
+      "loss": 1.7508,
+      "step": 265
+    },
+    {
+      "epoch": 0.3310516490354698,
+      "grad_norm": 3.26595401763916,
+      "learning_rate": 4.976937269372694e-06,
+      "loss": 1.7248,
+      "step": 266
+    },
+    {
+      "epoch": 0.33229620410703176,
+      "grad_norm": 2.929210662841797,
+      "learning_rate": 4.974630996309964e-06,
+      "loss": 1.697,
+      "step": 267
+    },
+    {
+      "epoch": 0.33354075917859366,
+      "grad_norm": 3.1526286602020264,
+      "learning_rate": 4.972324723247233e-06,
+      "loss": 1.7385,
+      "step": 268
+    },
+    {
+      "epoch": 0.33478531425015556,
+      "grad_norm": 3.001619577407837,
+      "learning_rate": 4.970018450184502e-06,
+      "loss": 1.7072,
+      "step": 269
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "grad_norm": 3.2464189529418945,
+      "learning_rate": 4.9677121771217715e-06,
+      "loss": 1.7361,
+      "step": 270
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "eval_loss": 1.860226035118103,
+      "eval_runtime": 46.6881,
+      "eval_samples_per_second": 21.419,
+      "eval_steps_per_second": 0.9,
+      "step": 270
+    },
+    {
+      "epoch": 0.3372744243932794,
+      "grad_norm": 3.2104530334472656,
+      "learning_rate": 4.965405904059041e-06,
+      "loss": 1.7171,
+      "step": 271
+    },
+    {
+      "epoch": 0.33851897946484133,
+      "grad_norm": 3.146847724914551,
+      "learning_rate": 4.96309963099631e-06,
+      "loss": 1.7311,
+      "step": 272
+    },
+    {
+      "epoch": 0.33976353453640323,
+      "grad_norm": 3.1431286334991455,
+      "learning_rate": 4.96079335793358e-06,
+      "loss": 1.7345,
+      "step": 273
+    },
+    {
+      "epoch": 0.34100808960796514,
+      "grad_norm": 2.847163677215576,
+      "learning_rate": 4.958487084870849e-06,
+      "loss": 1.7262,
+      "step": 274
+    },
+    {
+      "epoch": 0.3422526446795271,
+      "grad_norm": 3.1987810134887695,
+      "learning_rate": 4.956180811808119e-06,
+      "loss": 1.7945,
+      "step": 275
+    },
+    {
+      "epoch": 0.343497199751089,
+      "grad_norm": 2.7475385665893555,
+      "learning_rate": 4.953874538745388e-06,
+      "loss": 1.76,
+      "step": 276
+    },
+    {
+      "epoch": 0.3447417548226509,
+      "grad_norm": 3.0427663326263428,
+      "learning_rate": 4.9515682656826574e-06,
+      "loss": 1.7324,
+      "step": 277
+    },
+    {
+      "epoch": 0.3459863098942128,
+      "grad_norm": 3.605212688446045,
+      "learning_rate": 4.949261992619927e-06,
+      "loss": 1.7712,
+      "step": 278
+    },
+    {
+      "epoch": 0.3472308649657747,
+      "grad_norm": 3.0564935207366943,
+      "learning_rate": 4.946955719557196e-06,
+      "loss": 1.6792,
+      "step": 279
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "grad_norm": 3.1069741249084473,
+      "learning_rate": 4.944649446494466e-06,
+      "loss": 1.7145,
+      "step": 280
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "eval_loss": 1.8537051677703857,
+      "eval_runtime": 47.4813,
+      "eval_samples_per_second": 21.061,
+      "eval_steps_per_second": 0.885,
+      "step": 280
+    },
+    {
+      "epoch": 0.3497199751088986,
+      "grad_norm": 2.8021512031555176,
+      "learning_rate": 4.942343173431734e-06,
+      "loss": 1.7145,
+      "step": 281
+    },
+    {
+      "epoch": 0.3509645301804605,
+      "grad_norm": 3.5549023151397705,
+      "learning_rate": 4.940036900369004e-06,
+      "loss": 1.7642,
+      "step": 282
+    },
+    {
+      "epoch": 0.3522090852520224,
+      "grad_norm": 2.7648985385894775,
+      "learning_rate": 4.937730627306274e-06,
+      "loss": 1.7255,
+      "step": 283
+    },
+    {
+      "epoch": 0.35345364032358434,
+      "grad_norm": 3.0815863609313965,
+      "learning_rate": 4.9354243542435426e-06,
+      "loss": 1.7055,
+      "step": 284
+    },
+    {
+      "epoch": 0.35469819539514624,
+      "grad_norm": 2.9009227752685547,
+      "learning_rate": 4.933118081180812e-06,
+      "loss": 1.7019,
+      "step": 285
+    },
+    {
+      "epoch": 0.35594275046670815,
+      "grad_norm": 2.618429660797119,
+      "learning_rate": 4.930811808118081e-06,
+      "loss": 1.6318,
+      "step": 286
+    },
+    {
+      "epoch": 0.35718730553827005,
+      "grad_norm": 3.368230104446411,
+      "learning_rate": 4.928505535055351e-06,
+      "loss": 1.7304,
+      "step": 287
+    },
+    {
+      "epoch": 0.358431860609832,
+      "grad_norm": 3.256889820098877,
+      "learning_rate": 4.92619926199262e-06,
+      "loss": 1.7019,
+      "step": 288
+    },
+    {
+      "epoch": 0.3596764156813939,
+      "grad_norm": 2.9366304874420166,
+      "learning_rate": 4.92389298892989e-06,
+      "loss": 1.734,
+      "step": 289
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "grad_norm": 3.0393142700195312,
+      "learning_rate": 4.921586715867159e-06,
+      "loss": 1.7504,
+      "step": 290
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "eval_loss": 1.8359886407852173,
+      "eval_runtime": 49.7622,
+      "eval_samples_per_second": 20.096,
+      "eval_steps_per_second": 0.844,
+      "step": 290
+    },
+    {
+      "epoch": 0.3621655258245177,
+      "grad_norm": 2.7288589477539062,
+      "learning_rate": 4.9192804428044285e-06,
+      "loss": 1.7004,
+      "step": 291
+    },
+    {
+      "epoch": 0.3634100808960797,
+      "grad_norm": 3.2793378829956055,
+      "learning_rate": 4.916974169741698e-06,
+      "loss": 1.7028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3646546359676416,
+      "grad_norm": 2.960880756378174,
+      "learning_rate": 4.914667896678967e-06,
+      "loss": 1.6759,
+      "step": 293
+    },
+    {
+      "epoch": 0.3658991910392035,
+      "grad_norm": 2.836421012878418,
+      "learning_rate": 4.912361623616237e-06,
+      "loss": 1.6475,
+      "step": 294
+    },
+    {
+      "epoch": 0.3671437461107654,
+      "grad_norm": 2.9495465755462646,
+      "learning_rate": 4.910055350553506e-06,
+      "loss": 1.7076,
+      "step": 295
+    },
+    {
+      "epoch": 0.3683883011823273,
+      "grad_norm": 2.954730272293091,
+      "learning_rate": 4.907749077490776e-06,
+      "loss": 1.7261,
+      "step": 296
+    },
+    {
+      "epoch": 0.36963285625388925,
+      "grad_norm": 2.8167543411254883,
+      "learning_rate": 4.905442804428044e-06,
+      "loss": 1.6939,
+      "step": 297
+    },
+    {
+      "epoch": 0.37087741132545116,
+      "grad_norm": 2.857316017150879,
+      "learning_rate": 4.903136531365314e-06,
+      "loss": 1.7362,
+      "step": 298
+    },
+    {
+      "epoch": 0.37212196639701306,
+      "grad_norm": 2.8612918853759766,
+      "learning_rate": 4.900830258302584e-06,
+      "loss": 1.6489,
+      "step": 299
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "grad_norm": 2.9618914127349854,
+      "learning_rate": 4.898523985239853e-06,
+      "loss": 1.718,
+      "step": 300
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "eval_loss": 1.827571988105774,
+      "eval_runtime": 46.4552,
+      "eval_samples_per_second": 21.526,
+      "eval_steps_per_second": 0.904,
+      "step": 300
+    },
+    {
+      "epoch": 0.3746110765401369,
+      "grad_norm": 2.8706953525543213,
+      "learning_rate": 4.896217712177122e-06,
+      "loss": 1.6925,
+      "step": 301
+    },
+    {
+      "epoch": 0.3758556316116988,
+      "grad_norm": 3.2565090656280518,
+      "learning_rate": 4.893911439114391e-06,
+      "loss": 1.7369,
+      "step": 302
+    },
+    {
+      "epoch": 0.37710018668326073,
+      "grad_norm": 2.928858995437622,
+      "learning_rate": 4.891605166051661e-06,
+      "loss": 1.6882,
+      "step": 303
+    },
+    {
+      "epoch": 0.37834474175482263,
+      "grad_norm": 3.042314052581787,
+      "learning_rate": 4.88929889298893e-06,
+      "loss": 1.6466,
+      "step": 304
+    },
+    {
+      "epoch": 0.3795892968263846,
+      "grad_norm": 2.7509143352508545,
+      "learning_rate": 4.8869926199262e-06,
+      "loss": 1.685,
+      "step": 305
+    },
+    {
+      "epoch": 0.3808338518979465,
+      "grad_norm": 2.9601588249206543,
+      "learning_rate": 4.884686346863469e-06,
+      "loss": 1.6772,
+      "step": 306
+    },
+    {
+      "epoch": 0.3820784069695084,
+      "grad_norm": 2.618608236312866,
+      "learning_rate": 4.8823800738007384e-06,
+      "loss": 1.7127,
+      "step": 307
+    },
+    {
+      "epoch": 0.3833229620410703,
+      "grad_norm": 3.047581195831299,
+      "learning_rate": 4.880073800738008e-06,
+      "loss": 1.7174,
+      "step": 308
+    },
+    {
+      "epoch": 0.3845675171126322,
+      "grad_norm": 2.87735652923584,
+      "learning_rate": 4.877767527675277e-06,
+      "loss": 1.6677,
+      "step": 309
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "grad_norm": 2.7169721126556396,
+      "learning_rate": 4.875461254612546e-06,
+      "loss": 1.7486,
+      "step": 310
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "eval_loss": 1.8221794366836548,
+      "eval_runtime": 43.1681,
+      "eval_samples_per_second": 23.165,
+      "eval_steps_per_second": 0.973,
+      "step": 310
+    },
+    {
+      "epoch": 0.38705662725575607,
+      "grad_norm": 2.7094991207122803,
+      "learning_rate": 4.873154981549816e-06,
+      "loss": 1.7348,
+      "step": 311
+    },
+    {
+      "epoch": 0.388301182327318,
+      "grad_norm": 2.8989078998565674,
+      "learning_rate": 4.8708487084870856e-06,
+      "loss": 1.7352,
+      "step": 312
+    },
+    {
+      "epoch": 0.3895457373988799,
+      "grad_norm": 2.9565601348876953,
+      "learning_rate": 4.868542435424355e-06,
+      "loss": 1.7063,
+      "step": 313
+    },
+    {
+      "epoch": 0.39079029247044184,
+      "grad_norm": 2.8354082107543945,
+      "learning_rate": 4.8662361623616235e-06,
+      "loss": 1.6808,
+      "step": 314
+    },
+    {
+      "epoch": 0.39203484754200374,
+      "grad_norm": 2.8287479877471924,
+      "learning_rate": 4.863929889298894e-06,
+      "loss": 1.6731,
+      "step": 315
+    },
+    {
+      "epoch": 0.39327940261356564,
+      "grad_norm": 2.6537821292877197,
+      "learning_rate": 4.861623616236163e-06,
+      "loss": 1.695,
+      "step": 316
+    },
+    {
+      "epoch": 0.39452395768512755,
+      "grad_norm": 2.7135190963745117,
+      "learning_rate": 4.859317343173432e-06,
+      "loss": 1.6897,
+      "step": 317
+    },
+    {
+      "epoch": 0.3957685127566895,
+      "grad_norm": 3.1019129753112793,
+      "learning_rate": 4.857011070110701e-06,
+      "loss": 1.7219,
+      "step": 318
+    },
+    {
+      "epoch": 0.3970130678282514,
+      "grad_norm": 2.945737838745117,
+      "learning_rate": 4.854704797047971e-06,
+      "loss": 1.7316,
+      "step": 319
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "grad_norm": 2.7668251991271973,
+      "learning_rate": 4.85239852398524e-06,
+      "loss": 1.6461,
+      "step": 320
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "eval_loss": 1.8210570812225342,
+      "eval_runtime": 46.2552,
+      "eval_samples_per_second": 21.619,
+      "eval_steps_per_second": 0.908,
+      "step": 320
+    },
+    {
+      "epoch": 0.3995021779713752,
+      "grad_norm": 2.738109588623047,
+      "learning_rate": 4.8500922509225095e-06,
+      "loss": 1.6907,
+      "step": 321
+    },
+    {
+      "epoch": 0.4007467330429372,
+      "grad_norm": 3.0195062160491943,
+      "learning_rate": 4.847785977859779e-06,
+      "loss": 1.7099,
+      "step": 322
+    },
+    {
+      "epoch": 0.4019912881144991,
+      "grad_norm": 2.8672404289245605,
+      "learning_rate": 4.845479704797048e-06,
+      "loss": 1.6509,
+      "step": 323
+    },
+    {
+      "epoch": 0.403235843186061,
+      "grad_norm": 2.9041008949279785,
+      "learning_rate": 4.843173431734318e-06,
+      "loss": 1.5818,
+      "step": 324
+    },
+    {
+      "epoch": 0.4044803982576229,
+      "grad_norm": 2.9105985164642334,
+      "learning_rate": 4.840867158671587e-06,
+      "loss": 1.7002,
+      "step": 325
+    },
+    {
+      "epoch": 0.4057249533291848,
+      "grad_norm": 2.749009847640991,
+      "learning_rate": 4.838560885608857e-06,
+      "loss": 1.7021,
+      "step": 326
+    },
+    {
+      "epoch": 0.40696950840074675,
+      "grad_norm": 2.6998822689056396,
+      "learning_rate": 4.836254612546126e-06,
+      "loss": 1.6853,
+      "step": 327
+    },
+    {
+      "epoch": 0.40821406347230865,
+      "grad_norm": 2.678201675415039,
+      "learning_rate": 4.8339483394833955e-06,
+      "loss": 1.6934,
+      "step": 328
+    },
+    {
+      "epoch": 0.40945861854387056,
+      "grad_norm": 2.774845600128174,
+      "learning_rate": 4.831642066420665e-06,
+      "loss": 1.657,
+      "step": 329
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "grad_norm": 2.6363000869750977,
+      "learning_rate": 4.8293357933579335e-06,
+      "loss": 1.6555,
+      "step": 330
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "eval_loss": 1.8126581907272339,
+      "eval_runtime": 52.7717,
+      "eval_samples_per_second": 18.95,
+      "eval_steps_per_second": 0.796,
+      "step": 330
+    },
+    {
+      "epoch": 0.4119477286869944,
+      "grad_norm": 3.011659622192383,
+      "learning_rate": 4.827029520295204e-06,
+      "loss": 1.7202,
+      "step": 331
+    },
+    {
+      "epoch": 0.4131922837585563,
+      "grad_norm": 2.8322594165802,
+      "learning_rate": 4.824723247232473e-06,
+      "loss": 1.6736,
+      "step": 332
+    },
+    {
+      "epoch": 0.4144368388301182,
+      "grad_norm": 2.9360146522521973,
+      "learning_rate": 4.822416974169742e-06,
+      "loss": 1.7213,
+      "step": 333
+    },
+    {
+      "epoch": 0.41568139390168013,
+      "grad_norm": 2.810206413269043,
+      "learning_rate": 4.820110701107011e-06,
+      "loss": 1.6867,
+      "step": 334
+    },
+    {
+      "epoch": 0.4169259489732421,
+      "grad_norm": 2.6864097118377686,
+      "learning_rate": 4.817804428044281e-06,
+      "loss": 1.6854,
+      "step": 335
+    },
+    {
+      "epoch": 0.418170504044804,
+      "grad_norm": 2.7396061420440674,
+      "learning_rate": 4.81549815498155e-06,
+      "loss": 1.68,
+      "step": 336
+    },
+    {
+      "epoch": 0.4194150591163659,
+      "grad_norm": 2.602771520614624,
+      "learning_rate": 4.8131918819188194e-06,
+      "loss": 1.6269,
+      "step": 337
+    },
+    {
+      "epoch": 0.4206596141879278,
+      "grad_norm": 2.8174757957458496,
+      "learning_rate": 4.810885608856089e-06,
+      "loss": 1.6981,
+      "step": 338
+    },
+    {
+      "epoch": 0.42190416925948976,
+      "grad_norm": 3.2398130893707275,
+      "learning_rate": 4.808579335793358e-06,
+      "loss": 1.7061,
+      "step": 339
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "grad_norm": 2.7119736671447754,
+      "learning_rate": 4.806273062730628e-06,
+      "loss": 1.6875,
+      "step": 340
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "eval_loss": 1.7973068952560425,
+      "eval_runtime": 53.5791,
+      "eval_samples_per_second": 18.664,
+      "eval_steps_per_second": 0.784,
+      "step": 340
+    },
+    {
+      "epoch": 0.42439327940261357,
+      "grad_norm": 2.868533134460449,
+      "learning_rate": 4.803966789667897e-06,
+      "loss": 1.6967,
+      "step": 341
+    },
+    {
+      "epoch": 0.42563783447417547,
+      "grad_norm": 2.528083562850952,
+      "learning_rate": 4.8016605166051665e-06,
+      "loss": 1.6195,
+      "step": 342
+    },
+    {
+      "epoch": 0.4268823895457374,
+      "grad_norm": 2.7885115146636963,
+      "learning_rate": 4.799354243542436e-06,
+      "loss": 1.6197,
+      "step": 343
+    },
+    {
+      "epoch": 0.42812694461729933,
+      "grad_norm": 2.768247604370117,
+      "learning_rate": 4.797047970479705e-06,
+      "loss": 1.6271,
+      "step": 344
+    },
+    {
+      "epoch": 0.42937149968886124,
+      "grad_norm": 2.7710275650024414,
+      "learning_rate": 4.794741697416975e-06,
+      "loss": 1.6768,
+      "step": 345
+    },
+    {
+      "epoch": 0.43061605476042314,
+      "grad_norm": 2.968337297439575,
+      "learning_rate": 4.792435424354243e-06,
+      "loss": 1.6433,
+      "step": 346
+    },
+    {
+      "epoch": 0.43186060983198504,
+      "grad_norm": 2.5739808082580566,
+      "learning_rate": 4.790129151291514e-06,
+      "loss": 1.6327,
+      "step": 347
+    },
+    {
+      "epoch": 0.433105164903547,
+      "grad_norm": 2.7676408290863037,
+      "learning_rate": 4.787822878228783e-06,
+      "loss": 1.6178,
+      "step": 348
+    },
+    {
+      "epoch": 0.4343497199751089,
+      "grad_norm": 2.959059476852417,
+      "learning_rate": 4.7855166051660525e-06,
+      "loss": 1.6809,
+      "step": 349
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "grad_norm": 2.559967279434204,
+      "learning_rate": 4.783210332103321e-06,
+      "loss": 1.6704,
+      "step": 350
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "eval_loss": 1.8013949394226074,
+      "eval_runtime": 50.4649,
+      "eval_samples_per_second": 19.816,
+      "eval_steps_per_second": 0.832,
+      "step": 350
+    },
+    {
+      "epoch": 0.4368388301182327,
+      "grad_norm": 2.817901611328125,
+      "learning_rate": 4.7809040590405905e-06,
+      "loss": 1.6296,
+      "step": 351
+    },
+    {
+      "epoch": 0.43808338518979467,
+      "grad_norm": 2.7116055488586426,
+      "learning_rate": 4.778597785977861e-06,
+      "loss": 1.6652,
+      "step": 352
+    },
+    {
+      "epoch": 0.4393279402613566,
+      "grad_norm": 2.544750452041626,
+      "learning_rate": 4.776291512915129e-06,
+      "loss": 1.5668,
+      "step": 353
+    },
+    {
+      "epoch": 0.4405724953329185,
+      "grad_norm": 2.720534086227417,
+      "learning_rate": 4.773985239852399e-06,
+      "loss": 1.6152,
+      "step": 354
+    },
+    {
+      "epoch": 0.4418170504044804,
+      "grad_norm": 2.7270126342773438,
+      "learning_rate": 4.771678966789668e-06,
+      "loss": 1.5624,
+      "step": 355
+    },
+    {
+      "epoch": 0.44306160547604234,
+      "grad_norm": 2.7986624240875244,
+      "learning_rate": 4.769372693726938e-06,
+      "loss": 1.6621,
+      "step": 356
+    },
+    {
+      "epoch": 0.44430616054760425,
+      "grad_norm": 2.943107843399048,
+      "learning_rate": 4.767066420664207e-06,
+      "loss": 1.6713,
+      "step": 357
+    },
+    {
+      "epoch": 0.44555071561916615,
+      "grad_norm": 2.739898681640625,
+      "learning_rate": 4.7647601476014765e-06,
+      "loss": 1.6363,
+      "step": 358
+    },
+    {
+      "epoch": 0.44679527069072805,
+      "grad_norm": 2.8729329109191895,
+      "learning_rate": 4.762453874538746e-06,
+      "loss": 1.6605,
+      "step": 359
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "grad_norm": 2.909029483795166,
+      "learning_rate": 4.760147601476015e-06,
+      "loss": 1.6277,
+      "step": 360
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "eval_loss": 1.8110179901123047,
+      "eval_runtime": 47.924,
+      "eval_samples_per_second": 20.866,
+      "eval_steps_per_second": 0.876,
+      "step": 360
+    },
+    {
+      "epoch": 0.4492843808338519,
+      "grad_norm": 3.3826682567596436,
+      "learning_rate": 4.757841328413285e-06,
+      "loss": 1.7155,
+      "step": 361
+    },
+    {
+      "epoch": 0.4505289359054138,
+      "grad_norm": 3.0586729049682617,
+      "learning_rate": 4.755535055350554e-06,
+      "loss": 1.5991,
+      "step": 362
+    },
+    {
+      "epoch": 0.4517734909769757,
+      "grad_norm": 2.645576238632202,
+      "learning_rate": 4.753228782287823e-06,
+      "loss": 1.6504,
+      "step": 363
+    },
+    {
+      "epoch": 0.4530180460485376,
+      "grad_norm": 2.705967664718628,
+      "learning_rate": 4.750922509225093e-06,
+      "loss": 1.6761,
+      "step": 364
+    },
+    {
+      "epoch": 0.4542626011200996,
+      "grad_norm": 2.7651922702789307,
+      "learning_rate": 4.748616236162362e-06,
+      "loss": 1.61,
+      "step": 365
+    },
+    {
+      "epoch": 0.4555071561916615,
+      "grad_norm": 2.5654966831207275,
+      "learning_rate": 4.746309963099631e-06,
+      "loss": 1.6335,
+      "step": 366
+    },
+    {
+      "epoch": 0.4567517112632234,
+      "grad_norm": 2.6606173515319824,
+      "learning_rate": 4.7440036900369e-06,
+      "loss": 1.6207,
+      "step": 367
+    },
+    {
+      "epoch": 0.4579962663347853,
+      "grad_norm": 2.724755048751831,
+      "learning_rate": 4.741697416974171e-06,
+      "loss": 1.6329,
+      "step": 368
+    },
+    {
+      "epoch": 0.45924082140634725,
+      "grad_norm": 2.7019615173339844,
+      "learning_rate": 4.739391143911439e-06,
+      "loss": 1.6763,
+      "step": 369
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "grad_norm": 2.6594254970550537,
+      "learning_rate": 4.737084870848709e-06,
+      "loss": 1.6174,
+      "step": 370
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "eval_loss": 1.7924141883850098,
+      "eval_runtime": 53.0349,
+      "eval_samples_per_second": 18.855,
+      "eval_steps_per_second": 0.792,
+      "step": 370
+    },
+    {
+      "epoch": 0.46172993154947106,
+      "grad_norm": 2.668421745300293,
+      "learning_rate": 4.734778597785978e-06,
+      "loss": 1.6421,
+      "step": 371
+    },
+    {
+      "epoch": 0.46297448662103297,
+      "grad_norm": 2.7148995399475098,
+      "learning_rate": 4.7324723247232475e-06,
+      "loss": 1.6635,
+      "step": 372
+    },
+    {
+      "epoch": 0.4642190416925949,
+      "grad_norm": 3.156646490097046,
+      "learning_rate": 4.730166051660517e-06,
+      "loss": 1.6679,
+      "step": 373
+    },
+    {
+      "epoch": 0.46546359676415683,
+      "grad_norm": 2.8170645236968994,
+      "learning_rate": 4.727859778597786e-06,
+      "loss": 1.6492,
+      "step": 374
+    },
+    {
+      "epoch": 0.46670815183571873,
+      "grad_norm": 2.9796457290649414,
+      "learning_rate": 4.725553505535056e-06,
+      "loss": 1.6025,
+      "step": 375
+    },
+    {
+      "epoch": 0.46795270690728064,
+      "grad_norm": 2.6493406295776367,
+      "learning_rate": 4.723247232472325e-06,
+      "loss": 1.6398,
+      "step": 376
+    },
+    {
+      "epoch": 0.46919726197884254,
+      "grad_norm": 3.2643542289733887,
+      "learning_rate": 4.720940959409595e-06,
+      "loss": 1.6413,
+      "step": 377
+    },
+    {
+      "epoch": 0.4704418170504045,
+      "grad_norm": 2.7613179683685303,
+      "learning_rate": 4.718634686346864e-06,
+      "loss": 1.6251,
+      "step": 378
+    },
+    {
+      "epoch": 0.4716863721219664,
+      "grad_norm": 3.1216518878936768,
+      "learning_rate": 4.716328413284133e-06,
+      "loss": 1.6268,
+      "step": 379
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "grad_norm": 2.571647882461548,
+      "learning_rate": 4.714022140221403e-06,
+      "loss": 1.6245,
+      "step": 380
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "eval_loss": 1.7875893115997314,
+      "eval_runtime": 49.3058,
+      "eval_samples_per_second": 20.282,
+      "eval_steps_per_second": 0.852,
+      "step": 380
+    },
+    {
+      "epoch": 0.4741754822650902,
+      "grad_norm": 3.443906545639038,
+      "learning_rate": 4.711715867158672e-06,
+      "loss": 1.6799,
+      "step": 381
+    },
+    {
+      "epoch": 0.47542003733665217,
+      "grad_norm": 2.7887957096099854,
+      "learning_rate": 4.709409594095941e-06,
+      "loss": 1.6589,
+      "step": 382
+    },
+    {
+      "epoch": 0.47666459240821407,
+      "grad_norm": 3.2833151817321777,
+      "learning_rate": 4.70710332103321e-06,
+      "loss": 1.6064,
+      "step": 383
+    },
+    {
+      "epoch": 0.477909147479776,
+      "grad_norm": 2.6975057125091553,
+      "learning_rate": 4.704797047970481e-06,
+      "loss": 1.6029,
+      "step": 384
+    },
+    {
+      "epoch": 0.4791537025513379,
+      "grad_norm": 2.745246171951294,
+      "learning_rate": 4.702490774907749e-06,
+      "loss": 1.7127,
+      "step": 385
+    },
+    {
+      "epoch": 0.48039825762289984,
+      "grad_norm": 3.0522775650024414,
+      "learning_rate": 4.700184501845019e-06,
+      "loss": 1.6508,
+      "step": 386
+    },
+    {
+      "epoch": 0.48164281269446174,
+      "grad_norm": 2.689218044281006,
+      "learning_rate": 4.697878228782288e-06,
+      "loss": 1.6971,
+      "step": 387
+    },
+    {
+      "epoch": 0.48288736776602365,
+      "grad_norm": 3.060361862182617,
+      "learning_rate": 4.6955719557195575e-06,
+      "loss": 1.6464,
+      "step": 388
+    },
+    {
+      "epoch": 0.48413192283758555,
+      "grad_norm": 2.872262477874756,
+      "learning_rate": 4.693265682656827e-06,
+      "loss": 1.6293,
+      "step": 389
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "grad_norm": 2.980050563812256,
+      "learning_rate": 4.690959409594096e-06,
+      "loss": 1.6435,
+      "step": 390
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "eval_loss": 1.7791496515274048,
+      "eval_runtime": 50.0299,
+      "eval_samples_per_second": 19.988,
+      "eval_steps_per_second": 0.839,
+      "step": 390
+    },
+    {
+      "epoch": 0.4866210329807094,
+      "grad_norm": 2.764827251434326,
+      "learning_rate": 4.688653136531366e-06,
+      "loss": 1.6279,
+      "step": 391
+    },
+    {
+      "epoch": 0.4878655880522713,
+      "grad_norm": 2.6912333965301514,
+      "learning_rate": 4.686346863468635e-06,
+      "loss": 1.7276,
+      "step": 392
+    },
+    {
+      "epoch": 0.4891101431238332,
+      "grad_norm": 2.8358330726623535,
+      "learning_rate": 4.6840405904059046e-06,
+      "loss": 1.6482,
+      "step": 393
+    },
+    {
+      "epoch": 0.4903546981953951,
+      "grad_norm": 2.6581575870513916,
+      "learning_rate": 4.681734317343174e-06,
+      "loss": 1.6853,
+      "step": 394
+    },
+    {
+      "epoch": 0.4915992532669571,
+      "grad_norm": 2.941425323486328,
+      "learning_rate": 4.6794280442804426e-06,
+      "loss": 1.6075,
+      "step": 395
+    },
+    {
+      "epoch": 0.492843808338519,
+      "grad_norm": 3.2334814071655273,
+      "learning_rate": 4.677121771217713e-06,
+      "loss": 1.6157,
+      "step": 396
+    },
+    {
+      "epoch": 0.4940883634100809,
+      "grad_norm": 2.984410524368286,
+      "learning_rate": 4.674815498154982e-06,
+      "loss": 1.6257,
+      "step": 397
+    },
+    {
+      "epoch": 0.4953329184816428,
+      "grad_norm": 3.0650789737701416,
+      "learning_rate": 4.672509225092252e-06,
+      "loss": 1.6573,
+      "step": 398
+    },
+    {
+      "epoch": 0.49657747355320475,
+      "grad_norm": 2.798664093017578,
+      "learning_rate": 4.67020295202952e-06,
+      "loss": 1.638,
+      "step": 399
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "grad_norm": 2.8266706466674805,
+      "learning_rate": 4.66789667896679e-06,
+      "loss": 1.6484,
+      "step": 400
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "eval_loss": 1.7714862823486328,
+      "eval_runtime": 48.0657,
+      "eval_samples_per_second": 20.805,
+      "eval_steps_per_second": 0.874,
+      "step": 400
+    },
+    {
+      "epoch": 0.49906658369632856,
+      "grad_norm": 2.9928388595581055,
+      "learning_rate": 4.66559040590406e-06,
+      "loss": 1.6721,
+      "step": 401
+    },
+    {
+      "epoch": 0.5003111387678905,
+      "grad_norm": 2.573514461517334,
+      "learning_rate": 4.6632841328413285e-06,
+      "loss": 1.679,
+      "step": 402
+    },
+    {
+      "epoch": 0.5015556938394524,
+      "grad_norm": 3.1736996173858643,
+      "learning_rate": 4.660977859778598e-06,
+      "loss": 1.7124,
+      "step": 403
+    },
+    {
+      "epoch": 0.5028002489110143,
+      "grad_norm": 2.858849287033081,
+      "learning_rate": 4.658671586715867e-06,
+      "loss": 1.6622,
+      "step": 404
+    },
+    {
+      "epoch": 0.5040448039825762,
+      "grad_norm": 2.825698137283325,
+      "learning_rate": 4.656365313653137e-06,
+      "loss": 1.6464,
+      "step": 405
+    },
+    {
+      "epoch": 0.5052893590541382,
+      "grad_norm": 3.783891439437866,
+      "learning_rate": 4.654059040590406e-06,
+      "loss": 1.6665,
+      "step": 406
+    },
+    {
+      "epoch": 0.5065339141257,
+      "grad_norm": 2.789813756942749,
+      "learning_rate": 4.651752767527676e-06,
+      "loss": 1.6064,
+      "step": 407
+    },
+    {
+      "epoch": 0.507778469197262,
+      "grad_norm": 3.3106348514556885,
+      "learning_rate": 4.649446494464945e-06,
+      "loss": 1.6542,
+      "step": 408
+    },
+    {
+      "epoch": 0.5090230242688238,
+      "grad_norm": 2.8805112838745117,
+      "learning_rate": 4.6471402214022145e-06,
+      "loss": 1.6058,
+      "step": 409
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "grad_norm": 3.30898118019104,
+      "learning_rate": 4.644833948339484e-06,
+      "loss": 1.5933,
+      "step": 410
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "eval_loss": 1.7787818908691406,
+      "eval_runtime": 44.0876,
+      "eval_samples_per_second": 22.682,
+      "eval_steps_per_second": 0.953,
+      "step": 410
+    },
+    {
+      "epoch": 0.5115121344119478,
+      "grad_norm": 3.3563625812530518,
+      "learning_rate": 4.642527675276753e-06,
+      "loss": 1.6751,
+      "step": 411
+    },
+    {
+      "epoch": 0.5127566894835096,
+      "grad_norm": 2.968120574951172,
+      "learning_rate": 4.640221402214023e-06,
+      "loss": 1.663,
+      "step": 412
+    },
+    {
+      "epoch": 0.5140012445550716,
+      "grad_norm": 3.310248374938965,
+      "learning_rate": 4.637915129151292e-06,
+      "loss": 1.658,
+      "step": 413
+    },
+    {
+      "epoch": 0.5152457996266335,
+      "grad_norm": 2.7011170387268066,
+      "learning_rate": 4.635608856088562e-06,
+      "loss": 1.6894,
+      "step": 414
+    },
+    {
+      "epoch": 0.5164903546981954,
+      "grad_norm": 2.804901361465454,
+      "learning_rate": 4.63330258302583e-06,
+      "loss": 1.602,
+      "step": 415
+    },
+    {
+      "epoch": 0.5177349097697573,
+      "grad_norm": 3.2412750720977783,
+      "learning_rate": 4.6309963099631e-06,
+      "loss": 1.6158,
+      "step": 416
+    },
+    {
+      "epoch": 0.5189794648413192,
+      "grad_norm": 2.699909210205078,
+      "learning_rate": 4.62869003690037e-06,
+      "loss": 1.5842,
+      "step": 417
+    },
+    {
+      "epoch": 0.5202240199128811,
+      "grad_norm": 3.065929412841797,
+      "learning_rate": 4.6263837638376384e-06,
+      "loss": 1.6434,
+      "step": 418
+    },
+    {
+      "epoch": 0.5214685749844431,
+      "grad_norm": 2.895305633544922,
+      "learning_rate": 4.624077490774908e-06,
+      "loss": 1.5969,
+      "step": 419
+    },
+    {
+      "epoch": 0.522713130056005,
+      "grad_norm": 3.1590664386749268,
+      "learning_rate": 4.621771217712177e-06,
+      "loss": 1.6211,
+      "step": 420
+    },
+    {
+      "epoch": 0.522713130056005,
+      "eval_loss": 1.7674189805984497,
+      "eval_runtime": 53.328,
+      "eval_samples_per_second": 18.752,
+      "eval_steps_per_second": 0.788,
+      "step": 420
+    },
+    {
+      "epoch": 0.5239576851275669,
+      "grad_norm": 2.7126312255859375,
+      "learning_rate": 4.619464944649447e-06,
+      "loss": 1.6246,
+      "step": 421
+    },
+    {
+      "epoch": 0.5252022401991289,
+      "grad_norm": 2.918508768081665,
+      "learning_rate": 4.617158671586716e-06,
+      "loss": 1.6937,
+      "step": 422
+    },
+    {
+      "epoch": 0.5264467952706907,
+      "grad_norm": 3.008610725402832,
+      "learning_rate": 4.6148523985239856e-06,
+      "loss": 1.6864,
+      "step": 423
+    },
+    {
+      "epoch": 0.5276913503422527,
+      "grad_norm": 2.6612043380737305,
+      "learning_rate": 4.612546125461255e-06,
+      "loss": 1.5785,
+      "step": 424
+    },
+    {
+      "epoch": 0.5289359054138145,
+      "grad_norm": 2.850679874420166,
+      "learning_rate": 4.610239852398524e-06,
+      "loss": 1.5899,
+      "step": 425
+    },
+    {
+      "epoch": 0.5301804604853765,
+      "grad_norm": 2.9205031394958496,
+      "learning_rate": 4.607933579335794e-06,
+      "loss": 1.5867,
+      "step": 426
+    },
+    {
+      "epoch": 0.5314250155569384,
+      "grad_norm": 2.929532527923584,
+      "learning_rate": 4.605627306273063e-06,
+      "loss": 1.6182,
+      "step": 427
+    },
+    {
+      "epoch": 0.5326695706285003,
+      "grad_norm": 2.874936103820801,
+      "learning_rate": 4.603321033210333e-06,
+      "loss": 1.5789,
+      "step": 428
+    },
+    {
+      "epoch": 0.5339141257000622,
+      "grad_norm": 2.8703525066375732,
+      "learning_rate": 4.601014760147602e-06,
+      "loss": 1.5859,
+      "step": 429
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "grad_norm": 2.7076902389526367,
+      "learning_rate": 4.5987084870848715e-06,
+      "loss": 1.644,
+      "step": 430
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "eval_loss": 1.7592095136642456,
+      "eval_runtime": 47.9438,
+      "eval_samples_per_second": 20.858,
+      "eval_steps_per_second": 0.876,
+      "step": 430
+    },
+    {
+      "epoch": 0.536403235843186,
+      "grad_norm": 2.839160680770874,
+      "learning_rate": 4.59640221402214e-06,
+      "loss": 1.6357,
+      "step": 431
+    },
+    {
+      "epoch": 0.537647790914748,
+      "grad_norm": 2.674025535583496,
+      "learning_rate": 4.5940959409594095e-06,
+      "loss": 1.676,
+      "step": 432
+    },
+    {
+      "epoch": 0.5388923459863099,
+      "grad_norm": 3.0619537830352783,
+      "learning_rate": 4.59178966789668e-06,
+      "loss": 1.5617,
+      "step": 433
+    },
+    {
+      "epoch": 0.5401369010578718,
+      "grad_norm": 3.058418035507202,
+      "learning_rate": 4.589483394833948e-06,
+      "loss": 1.5951,
+      "step": 434
+    },
+    {
+      "epoch": 0.5413814561294338,
+      "grad_norm": 3.0046396255493164,
+      "learning_rate": 4.587177121771218e-06,
+      "loss": 1.6473,
+      "step": 435
+    },
+    {
+      "epoch": 0.5426260112009956,
+      "grad_norm": 2.7736752033233643,
+      "learning_rate": 4.584870848708487e-06,
+      "loss": 1.6262,
+      "step": 436
+    },
+    {
+      "epoch": 0.5438705662725576,
+      "grad_norm": 2.7804994583129883,
+      "learning_rate": 4.5825645756457575e-06,
+      "loss": 1.6886,
+      "step": 437
+    },
+    {
+      "epoch": 0.5451151213441194,
+      "grad_norm": 3.0717954635620117,
+      "learning_rate": 4.580258302583026e-06,
+      "loss": 1.6287,
+      "step": 438
+    },
+    {
+      "epoch": 0.5463596764156814,
+      "grad_norm": 2.800243854522705,
+      "learning_rate": 4.5779520295202955e-06,
+      "loss": 1.6169,
+      "step": 439
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "grad_norm": 2.9581644535064697,
+      "learning_rate": 4.575645756457565e-06,
+      "loss": 1.6617,
+      "step": 440
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "eval_loss": 1.7642868757247925,
+      "eval_runtime": 51.2646,
+      "eval_samples_per_second": 19.507,
+      "eval_steps_per_second": 0.819,
+      "step": 440
+    },
+    {
+      "epoch": 0.5488487865588052,
+      "grad_norm": 3.0067460536956787,
+      "learning_rate": 4.573339483394834e-06,
+      "loss": 1.6482,
+      "step": 441
+    },
+    {
+      "epoch": 0.5500933416303672,
+      "grad_norm": 2.9075019359588623,
+      "learning_rate": 4.571033210332104e-06,
+      "loss": 1.6595,
+      "step": 442
+    },
+    {
+      "epoch": 0.551337896701929,
+      "grad_norm": 2.7788755893707275,
+      "learning_rate": 4.568726937269373e-06,
+      "loss": 1.5733,
+      "step": 443
+    },
+    {
+      "epoch": 0.552582451773491,
+      "grad_norm": 2.7775425910949707,
+      "learning_rate": 4.566420664206643e-06,
+      "loss": 1.6925,
+      "step": 444
+    },
+    {
+      "epoch": 0.5538270068450529,
+      "grad_norm": 2.6949567794799805,
+      "learning_rate": 4.564114391143912e-06,
+      "loss": 1.5738,
+      "step": 445
+    },
+    {
+      "epoch": 0.5550715619166148,
+      "grad_norm": 2.7093007564544678,
+      "learning_rate": 4.5618081180811814e-06,
+      "loss": 1.5848,
+      "step": 446
+    },
+    {
+      "epoch": 0.5563161169881767,
+      "grad_norm": 2.6920173168182373,
+      "learning_rate": 4.559501845018451e-06,
+      "loss": 1.6092,
+      "step": 447
+    },
+    {
+      "epoch": 0.5575606720597387,
+      "grad_norm": 2.733731985092163,
+      "learning_rate": 4.5571955719557194e-06,
+      "loss": 1.6056,
+      "step": 448
+    },
+    {
+      "epoch": 0.5588052271313005,
+      "grad_norm": 2.9156274795532227,
+      "learning_rate": 4.55488929889299e-06,
+      "loss": 1.6436,
+      "step": 449
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "grad_norm": 2.8802616596221924,
+      "learning_rate": 4.552583025830259e-06,
+      "loss": 1.6309,
+      "step": 450
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "eval_loss": 1.7486340999603271,
+      "eval_runtime": 47.2219,
+      "eval_samples_per_second": 21.177,
+      "eval_steps_per_second": 0.889,
+      "step": 450
+    },
+    {
+      "epoch": 0.5612943372744243,
+      "grad_norm": 2.729743242263794,
+      "learning_rate": 4.550276752767528e-06,
+      "loss": 1.7171,
+      "step": 451
+    },
+    {
+      "epoch": 0.5625388923459863,
+      "grad_norm": 2.800049066543579,
+      "learning_rate": 4.547970479704797e-06,
+      "loss": 1.6098,
+      "step": 452
+    },
+    {
+      "epoch": 0.5637834474175483,
+      "grad_norm": 2.9659311771392822,
+      "learning_rate": 4.5456642066420666e-06,
+      "loss": 1.6646,
+      "step": 453
+    },
+    {
+      "epoch": 0.5650280024891101,
+      "grad_norm": 2.5417754650115967,
+      "learning_rate": 4.543357933579336e-06,
+      "loss": 1.5847,
+      "step": 454
+    },
+    {
+      "epoch": 0.5662725575606721,
+      "grad_norm": 2.8471338748931885,
+      "learning_rate": 4.541051660516605e-06,
+      "loss": 1.6235,
+      "step": 455
+    },
+    {
+      "epoch": 0.5675171126322339,
+      "grad_norm": 2.7039637565612793,
+      "learning_rate": 4.538745387453875e-06,
+      "loss": 1.5613,
+      "step": 456
+    },
+    {
+      "epoch": 0.5687616677037959,
+      "grad_norm": 3.0383341312408447,
+      "learning_rate": 4.536439114391144e-06,
+      "loss": 1.6485,
+      "step": 457
+    },
+    {
+      "epoch": 0.5700062227753578,
+      "grad_norm": 2.528388261795044,
+      "learning_rate": 4.534132841328414e-06,
+      "loss": 1.6117,
+      "step": 458
+    },
+    {
+      "epoch": 0.5712507778469197,
+      "grad_norm": 2.7623119354248047,
+      "learning_rate": 4.531826568265683e-06,
+      "loss": 1.6017,
+      "step": 459
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "grad_norm": 2.9213945865631104,
+      "learning_rate": 4.5295202952029525e-06,
+      "loss": 1.643,
+      "step": 460
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "eval_loss": 1.747575044631958,
+      "eval_runtime": 45.2844,
+      "eval_samples_per_second": 22.083,
+      "eval_steps_per_second": 0.927,
+      "step": 460
+    },
+    {
+      "epoch": 0.5737398879900436,
+      "grad_norm": 2.664275884628296,
+      "learning_rate": 4.527214022140222e-06,
+      "loss": 1.6154,
+      "step": 461
+    },
+    {
+      "epoch": 0.5749844430616055,
+      "grad_norm": 2.5891788005828857,
+      "learning_rate": 4.524907749077491e-06,
+      "loss": 1.5975,
+      "step": 462
+    },
+    {
+      "epoch": 0.5762289981331674,
+      "grad_norm": 2.8126487731933594,
+      "learning_rate": 4.522601476014761e-06,
+      "loss": 1.6306,
+      "step": 463
+    },
+    {
+      "epoch": 0.5774735532047293,
+      "grad_norm": 2.799391031265259,
+      "learning_rate": 4.520295202952029e-06,
+      "loss": 1.6139,
+      "step": 464
+    },
+    {
+      "epoch": 0.5787181082762912,
+      "grad_norm": 2.7323718070983887,
+      "learning_rate": 4.5179889298893e-06,
+      "loss": 1.632,
+      "step": 465
+    },
+    {
+      "epoch": 0.5799626633478532,
+      "grad_norm": 2.601386785507202,
+      "learning_rate": 4.515682656826569e-06,
+      "loss": 1.6139,
+      "step": 466
+    },
+    {
+      "epoch": 0.581207218419415,
+      "grad_norm": 2.562162399291992,
+      "learning_rate": 4.513376383763838e-06,
+      "loss": 1.5168,
+      "step": 467
+    },
+    {
+      "epoch": 0.582451773490977,
+      "grad_norm": 2.6912002563476562,
+      "learning_rate": 4.511070110701107e-06,
+      "loss": 1.6249,
+      "step": 468
+    },
+    {
+      "epoch": 0.583696328562539,
+      "grad_norm": 2.6971495151519775,
+      "learning_rate": 4.5087638376383765e-06,
+      "loss": 1.616,
+      "step": 469
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "grad_norm": 2.6898910999298096,
+      "learning_rate": 4.506457564575646e-06,
+      "loss": 1.614,
+      "step": 470
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "eval_loss": 1.7460769414901733,
+      "eval_runtime": 45.3055,
+      "eval_samples_per_second": 22.072,
+      "eval_steps_per_second": 0.927,
+      "step": 470
+    },
+    {
+      "epoch": 0.5861854387056628,
+      "grad_norm": 2.698180675506592,
+      "learning_rate": 4.504151291512915e-06,
+      "loss": 1.5993,
+      "step": 471
+    },
+    {
+      "epoch": 0.5874299937772246,
+      "grad_norm": 2.7125210762023926,
+      "learning_rate": 4.501845018450185e-06,
+      "loss": 1.6113,
+      "step": 472
+    },
+    {
+      "epoch": 0.5886745488487866,
+      "grad_norm": 2.754445791244507,
+      "learning_rate": 4.499538745387454e-06,
+      "loss": 1.6719,
+      "step": 473
+    },
+    {
+      "epoch": 0.5899191039203485,
+      "grad_norm": 2.6979362964630127,
+      "learning_rate": 4.497232472324724e-06,
+      "loss": 1.608,
+      "step": 474
+    },
+    {
+      "epoch": 0.5911636589919104,
+      "grad_norm": 2.7694528102874756,
+      "learning_rate": 4.494926199261993e-06,
+      "loss": 1.5695,
+      "step": 475
+    },
+    {
+      "epoch": 0.5924082140634723,
+      "grad_norm": 2.653353214263916,
+      "learning_rate": 4.4926199261992624e-06,
+      "loss": 1.5523,
+      "step": 476
+    },
+    {
+      "epoch": 0.5936527691350342,
+      "grad_norm": 2.6632070541381836,
+      "learning_rate": 4.490313653136532e-06,
+      "loss": 1.5675,
+      "step": 477
+    },
+    {
+      "epoch": 0.5948973242065961,
+      "grad_norm": 3.038543939590454,
+      "learning_rate": 4.488007380073801e-06,
+      "loss": 1.5692,
+      "step": 478
+    },
+    {
+      "epoch": 0.5961418792781581,
+      "grad_norm": 2.8123953342437744,
+      "learning_rate": 4.485701107011071e-06,
+      "loss": 1.6223,
+      "step": 479
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "grad_norm": 2.5845773220062256,
+      "learning_rate": 4.483394833948339e-06,
+      "loss": 1.5773,
+      "step": 480
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "eval_loss": 1.7419319152832031,
+      "eval_runtime": 44.6842,
+      "eval_samples_per_second": 22.379,
+      "eval_steps_per_second": 0.94,
+      "step": 480
+    },
+    {
+      "epoch": 0.5986309894212819,
+      "grad_norm": 3.069936752319336,
+      "learning_rate": 4.4810885608856096e-06,
+      "loss": 1.6323,
+      "step": 481
+    },
+    {
+      "epoch": 0.5998755444928439,
+      "grad_norm": 2.9085781574249268,
+      "learning_rate": 4.478782287822879e-06,
+      "loss": 1.5845,
+      "step": 482
+    },
+    {
+      "epoch": 0.6011200995644057,
+      "grad_norm": 2.8568010330200195,
+      "learning_rate": 4.4764760147601476e-06,
+      "loss": 1.5898,
+      "step": 483
+    },
+    {
+      "epoch": 0.6023646546359677,
+      "grad_norm": 3.089081048965454,
+      "learning_rate": 4.474169741697417e-06,
+      "loss": 1.6622,
+      "step": 484
+    },
+    {
+      "epoch": 0.6036092097075295,
+      "grad_norm": 2.8515470027923584,
+      "learning_rate": 4.471863468634686e-06,
+      "loss": 1.5859,
+      "step": 485
+    },
+    {
+      "epoch": 0.6048537647790915,
+      "grad_norm": 2.647765874862671,
+      "learning_rate": 4.469557195571957e-06,
+      "loss": 1.5926,
+      "step": 486
+    },
+    {
+      "epoch": 0.6060983198506534,
+      "grad_norm": 2.61676025390625,
+      "learning_rate": 4.467250922509225e-06,
+      "loss": 1.5684,
+      "step": 487
+    },
+    {
+      "epoch": 0.6073428749222153,
+      "grad_norm": 2.7127039432525635,
+      "learning_rate": 4.464944649446495e-06,
+      "loss": 1.5702,
+      "step": 488
+    },
+    {
+      "epoch": 0.6085874299937772,
+      "grad_norm": 2.9932055473327637,
+      "learning_rate": 4.462638376383764e-06,
+      "loss": 1.5976,
+      "step": 489
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "grad_norm": 2.6508774757385254,
+      "learning_rate": 4.4603321033210335e-06,
+      "loss": 1.586,
+      "step": 490
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "eval_loss": 1.7357326745986938,
+      "eval_runtime": 45.393,
+      "eval_samples_per_second": 22.03,
+      "eval_steps_per_second": 0.925,
+      "step": 490
+    },
+    {
+      "epoch": 0.611076540136901,
+      "grad_norm": 2.7626044750213623,
+      "learning_rate": 4.458025830258303e-06,
+      "loss": 1.5773,
+      "step": 491
+    },
+    {
+      "epoch": 0.612321095208463,
+      "grad_norm": 2.595003604888916,
+      "learning_rate": 4.455719557195572e-06,
+      "loss": 1.601,
+      "step": 492
+    },
+    {
+      "epoch": 0.6135656502800249,
+      "grad_norm": 2.595767021179199,
+      "learning_rate": 4.453413284132842e-06,
+      "loss": 1.6287,
+      "step": 493
+    },
+    {
+      "epoch": 0.6148102053515868,
+      "grad_norm": 2.755845308303833,
+      "learning_rate": 4.451107011070111e-06,
+      "loss": 1.593,
+      "step": 494
+    },
+    {
+      "epoch": 0.6160547604231488,
+      "grad_norm": 2.7302653789520264,
+      "learning_rate": 4.448800738007381e-06,
+      "loss": 1.5768,
+      "step": 495
+    },
+    {
+      "epoch": 0.6172993154947106,
+      "grad_norm": 2.6493024826049805,
+      "learning_rate": 4.446494464944649e-06,
+      "loss": 1.6059,
+      "step": 496
+    },
+    {
+      "epoch": 0.6185438705662726,
+      "grad_norm": 2.8633735179901123,
+      "learning_rate": 4.4441881918819195e-06,
+      "loss": 1.598,
+      "step": 497
+    },
+    {
+      "epoch": 0.6197884256378344,
+      "grad_norm": 2.65639066696167,
+      "learning_rate": 4.441881918819189e-06,
+      "loss": 1.6036,
+      "step": 498
+    },
+    {
+      "epoch": 0.6210329807093964,
+      "grad_norm": 2.6867759227752686,
+      "learning_rate": 4.439575645756458e-06,
+      "loss": 1.5676,
+      "step": 499
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "grad_norm": 2.7986443042755127,
+      "learning_rate": 4.437269372693727e-06,
+      "loss": 1.524,
+      "step": 500
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "eval_loss": 1.7449125051498413,
+      "eval_runtime": 43.9329,
+      "eval_samples_per_second": 22.762,
+      "eval_steps_per_second": 0.956,
+      "step": 500
+    },
+    {
+      "epoch": 0.6235220908525202,
+      "grad_norm": 2.665905714035034,
+      "learning_rate": 4.434963099630996e-06,
+      "loss": 1.5635,
+      "step": 501
+    },
+    {
+      "epoch": 0.6247666459240822,
+      "grad_norm": 2.902435541152954,
+      "learning_rate": 4.432656826568267e-06,
+      "loss": 1.6377,
+      "step": 502
+    },
+    {
+      "epoch": 0.6260112009956441,
+      "grad_norm": 2.6872262954711914,
+      "learning_rate": 4.430350553505535e-06,
+      "loss": 1.5553,
+      "step": 503
+    },
+    {
+      "epoch": 0.627255756067206,
+      "grad_norm": 2.676621913909912,
+      "learning_rate": 4.428044280442805e-06,
+      "loss": 1.6101,
+      "step": 504
+    },
+    {
+      "epoch": 0.6285003111387679,
+      "grad_norm": 2.756347179412842,
+      "learning_rate": 4.425738007380074e-06,
+      "loss": 1.5884,
+      "step": 505
+    },
+    {
+      "epoch": 0.6297448662103298,
+      "grad_norm": 2.856882333755493,
+      "learning_rate": 4.4234317343173434e-06,
+      "loss": 1.5635,
+      "step": 506
+    },
+    {
+      "epoch": 0.6309894212818917,
+      "grad_norm": 2.7708330154418945,
+      "learning_rate": 4.421125461254613e-06,
+      "loss": 1.6028,
+      "step": 507
+    },
+    {
+      "epoch": 0.6322339763534537,
+      "grad_norm": 2.8167600631713867,
+      "learning_rate": 4.418819188191882e-06,
+      "loss": 1.6026,
+      "step": 508
+    },
+    {
+      "epoch": 0.6334785314250155,
+      "grad_norm": 2.54194974899292,
+      "learning_rate": 4.416512915129152e-06,
+      "loss": 1.6001,
+      "step": 509
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "grad_norm": 2.683037519454956,
+      "learning_rate": 4.414206642066421e-06,
+      "loss": 1.6084,
+      "step": 510
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "eval_loss": 1.7376186847686768,
+      "eval_runtime": 42.3026,
+      "eval_samples_per_second": 23.639,
+      "eval_steps_per_second": 0.993,
+      "step": 510
+    },
+    {
+      "epoch": 0.6359676415681393,
+      "grad_norm": 2.82627010345459,
+      "learning_rate": 4.4119003690036905e-06,
+      "loss": 1.6258,
+      "step": 511
+    },
+    {
+      "epoch": 0.6372121966397013,
+      "grad_norm": 2.6139848232269287,
+      "learning_rate": 4.40959409594096e-06,
+      "loss": 1.6029,
+      "step": 512
+    },
+    {
+      "epoch": 0.6384567517112633,
+      "grad_norm": 2.646712303161621,
+      "learning_rate": 4.407287822878229e-06,
+      "loss": 1.5602,
+      "step": 513
+    },
+    {
+      "epoch": 0.6397013067828251,
+      "grad_norm": 2.910935163497925,
+      "learning_rate": 4.404981549815499e-06,
+      "loss": 1.5757,
+      "step": 514
+    },
+    {
+      "epoch": 0.6409458618543871,
+      "grad_norm": 2.6001148223876953,
+      "learning_rate": 4.402675276752768e-06,
+      "loss": 1.5857,
+      "step": 515
+    },
+    {
+      "epoch": 0.642190416925949,
+      "grad_norm": 2.8525235652923584,
+      "learning_rate": 4.400369003690037e-06,
+      "loss": 1.6092,
+      "step": 516
+    },
+    {
+      "epoch": 0.6434349719975109,
+      "grad_norm": 2.5335488319396973,
+      "learning_rate": 4.398062730627306e-06,
+      "loss": 1.5849,
+      "step": 517
+    },
+    {
+      "epoch": 0.6446795270690728,
+      "grad_norm": 2.5787103176116943,
+      "learning_rate": 4.3957564575645765e-06,
+      "loss": 1.634,
+      "step": 518
+    },
+    {
+      "epoch": 0.6459240821406347,
+      "grad_norm": 2.6188197135925293,
+      "learning_rate": 4.393450184501845e-06,
+      "loss": 1.5477,
+      "step": 519
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "grad_norm": 2.6548666954040527,
+      "learning_rate": 4.3911439114391145e-06,
+      "loss": 1.6133,
+      "step": 520
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "eval_loss": 1.7393039464950562,
+      "eval_runtime": 45.9234,
+      "eval_samples_per_second": 21.775,
+      "eval_steps_per_second": 0.915,
+      "step": 520
+    },
+    {
+      "epoch": 0.6484131922837586,
+      "grad_norm": 2.907928228378296,
+      "learning_rate": 4.388837638376384e-06,
+      "loss": 1.587,
+      "step": 521
+    },
+    {
+      "epoch": 0.6496577473553204,
+      "grad_norm": 2.7899692058563232,
+      "learning_rate": 4.386531365313653e-06,
+      "loss": 1.6126,
+      "step": 522
+    },
+    {
+      "epoch": 0.6509023024268824,
+      "grad_norm": 2.680147886276245,
+      "learning_rate": 4.384225092250923e-06,
+      "loss": 1.565,
+      "step": 523
+    },
+    {
+      "epoch": 0.6521468574984443,
+      "grad_norm": 2.6590754985809326,
+      "learning_rate": 4.381918819188192e-06,
+      "loss": 1.6127,
+      "step": 524
+    },
+    {
+      "epoch": 0.6533914125700062,
+      "grad_norm": 2.6638906002044678,
+      "learning_rate": 4.379612546125462e-06,
+      "loss": 1.5615,
+      "step": 525
+    },
+    {
+      "epoch": 0.6546359676415682,
+      "grad_norm": 2.6518194675445557,
+      "learning_rate": 4.377306273062731e-06,
+      "loss": 1.6202,
+      "step": 526
+    },
+    {
+      "epoch": 0.65588052271313,
+      "grad_norm": 2.5323619842529297,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 1.5305,
+      "step": 527
+    },
+    {
+      "epoch": 0.657125077784692,
+      "grad_norm": 2.6871442794799805,
+      "learning_rate": 4.37269372693727e-06,
+      "loss": 1.6043,
+      "step": 528
+    },
+    {
+      "epoch": 0.6583696328562539,
+      "grad_norm": 2.760746717453003,
+      "learning_rate": 4.370387453874539e-06,
+      "loss": 1.6059,
+      "step": 529
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "grad_norm": 2.780482769012451,
+      "learning_rate": 4.368081180811809e-06,
+      "loss": 1.6082,
+      "step": 530
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "eval_loss": 1.7219713926315308,
+      "eval_runtime": 51.8478,
+      "eval_samples_per_second": 19.287,
+      "eval_steps_per_second": 0.81,
+      "step": 530
+    },
+    {
+      "epoch": 0.6608587429993777,
+      "grad_norm": 2.605890989303589,
+      "learning_rate": 4.365774907749078e-06,
+      "loss": 1.5554,
+      "step": 531
+    },
+    {
+      "epoch": 0.6621032980709396,
+      "grad_norm": 2.731555461883545,
+      "learning_rate": 4.363468634686347e-06,
+      "loss": 1.606,
+      "step": 532
+    },
+    {
+      "epoch": 0.6633478531425016,
+      "grad_norm": 2.8356943130493164,
+      "learning_rate": 4.361162361623616e-06,
+      "loss": 1.5601,
+      "step": 533
+    },
+    {
+      "epoch": 0.6645924082140635,
+      "grad_norm": 2.7196593284606934,
+      "learning_rate": 4.3588560885608864e-06,
+      "loss": 1.5722,
+      "step": 534
+    },
+    {
+      "epoch": 0.6658369632856254,
+      "grad_norm": 2.621371269226074,
+      "learning_rate": 4.356549815498156e-06,
+      "loss": 1.6194,
+      "step": 535
+    },
+    {
+      "epoch": 0.6670815183571873,
+      "grad_norm": 2.653916120529175,
+      "learning_rate": 4.354243542435424e-06,
+      "loss": 1.5592,
+      "step": 536
+    },
+    {
+      "epoch": 0.6683260734287492,
+      "grad_norm": 2.88431453704834,
+      "learning_rate": 4.351937269372694e-06,
+      "loss": 1.6296,
+      "step": 537
+    },
+    {
+      "epoch": 0.6695706285003111,
+      "grad_norm": 2.667130470275879,
+      "learning_rate": 4.349630996309963e-06,
+      "loss": 1.5624,
+      "step": 538
+    },
+    {
+      "epoch": 0.6708151835718731,
+      "grad_norm": 2.6453566551208496,
+      "learning_rate": 4.347324723247233e-06,
+      "loss": 1.6025,
+      "step": 539
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "grad_norm": 3.032271146774292,
+      "learning_rate": 4.345018450184502e-06,
+      "loss": 1.6771,
+      "step": 540
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "eval_loss": 1.721895456314087,
+      "eval_runtime": 50.62,
+      "eval_samples_per_second": 19.755,
+      "eval_steps_per_second": 0.83,
+      "step": 540
+    },
+    {
+      "epoch": 0.6733042937149969,
+      "grad_norm": 2.8476362228393555,
+      "learning_rate": 4.3427121771217715e-06,
+      "loss": 1.5663,
+      "step": 541
+    },
+    {
+      "epoch": 0.6745488487865589,
+      "grad_norm": 2.615602970123291,
+      "learning_rate": 4.340405904059041e-06,
+      "loss": 1.5815,
+      "step": 542
+    },
+    {
+      "epoch": 0.6757934038581207,
+      "grad_norm": 3.221571683883667,
+      "learning_rate": 4.33809963099631e-06,
+      "loss": 1.5955,
+      "step": 543
+    },
+    {
+      "epoch": 0.6770379589296827,
+      "grad_norm": 2.602994203567505,
+      "learning_rate": 4.33579335793358e-06,
+      "loss": 1.5667,
+      "step": 544
+    },
+    {
+      "epoch": 0.6782825140012445,
+      "grad_norm": 2.9380433559417725,
+      "learning_rate": 4.333487084870848e-06,
+      "loss": 1.5651,
+      "step": 545
+    },
+    {
+      "epoch": 0.6795270690728065,
+      "grad_norm": 3.0689425468444824,
+      "learning_rate": 4.331180811808119e-06,
+      "loss": 1.6003,
+      "step": 546
+    },
+    {
+      "epoch": 0.6807716241443684,
+      "grad_norm": 2.770317316055298,
+      "learning_rate": 4.328874538745388e-06,
+      "loss": 1.5576,
+      "step": 547
+    },
+    {
+      "epoch": 0.6820161792159303,
+      "grad_norm": 2.8631815910339355,
+      "learning_rate": 4.3265682656826575e-06,
+      "loss": 1.5308,
+      "step": 548
+    },
+    {
+      "epoch": 0.6832607342874922,
+      "grad_norm": 2.785576820373535,
+      "learning_rate": 4.324261992619926e-06,
+      "loss": 1.6113,
+      "step": 549
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "grad_norm": 2.723919630050659,
+      "learning_rate": 4.321955719557196e-06,
+      "loss": 1.5606,
+      "step": 550
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "eval_loss": 1.721803069114685,
+      "eval_runtime": 51.6036,
+      "eval_samples_per_second": 19.378,
+      "eval_steps_per_second": 0.814,
+      "step": 550
+    },
+    {
+      "epoch": 0.685749844430616,
+      "grad_norm": 3.081198215484619,
+      "learning_rate": 4.319649446494466e-06,
+      "loss": 1.5643,
+      "step": 551
+    },
+    {
+      "epoch": 0.686994399502178,
+      "grad_norm": 2.757392168045044,
+      "learning_rate": 4.317343173431734e-06,
+      "loss": 1.5909,
+      "step": 552
+    },
+    {
+      "epoch": 0.6882389545737398,
+      "grad_norm": 3.038753032684326,
+      "learning_rate": 4.315036900369004e-06,
+      "loss": 1.6023,
+      "step": 553
+    },
+    {
+      "epoch": 0.6894835096453018,
+      "grad_norm": 2.8246452808380127,
+      "learning_rate": 4.312730627306273e-06,
+      "loss": 1.5548,
+      "step": 554
+    },
+    {
+      "epoch": 0.6907280647168638,
+      "grad_norm": 2.589320182800293,
+      "learning_rate": 4.310424354243543e-06,
+      "loss": 1.5424,
+      "step": 555
+    },
+    {
+      "epoch": 0.6919726197884256,
+      "grad_norm": 2.6413373947143555,
+      "learning_rate": 4.308118081180812e-06,
+      "loss": 1.6243,
+      "step": 556
+    },
+    {
+      "epoch": 0.6932171748599876,
+      "grad_norm": 2.764784336090088,
+      "learning_rate": 4.3058118081180815e-06,
+      "loss": 1.589,
+      "step": 557
+    },
+    {
+      "epoch": 0.6944617299315494,
+      "grad_norm": 2.806821823120117,
+      "learning_rate": 4.303505535055351e-06,
+      "loss": 1.621,
+      "step": 558
+    },
+    {
+      "epoch": 0.6957062850031114,
+      "grad_norm": 2.718017578125,
+      "learning_rate": 4.30119926199262e-06,
+      "loss": 1.5773,
+      "step": 559
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "grad_norm": 2.7620160579681396,
+      "learning_rate": 4.29889298892989e-06,
+      "loss": 1.5868,
+      "step": 560
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "eval_loss": 1.7152249813079834,
+      "eval_runtime": 52.0463,
+      "eval_samples_per_second": 19.214,
+      "eval_steps_per_second": 0.807,
+      "step": 560
+    },
+    {
+      "epoch": 0.6981953951462352,
+      "grad_norm": 2.66686749458313,
+      "learning_rate": 4.296586715867159e-06,
+      "loss": 1.534,
+      "step": 561
+    },
+    {
+      "epoch": 0.6994399502177971,
+      "grad_norm": 2.812577247619629,
+      "learning_rate": 4.2942804428044286e-06,
+      "loss": 1.5501,
+      "step": 562
+    },
+    {
+      "epoch": 0.7006845052893591,
+      "grad_norm": 2.578508138656616,
+      "learning_rate": 4.291974169741698e-06,
+      "loss": 1.6286,
+      "step": 563
+    },
+    {
+      "epoch": 0.701929060360921,
+      "grad_norm": 2.524249315261841,
+      "learning_rate": 4.289667896678967e-06,
+      "loss": 1.5719,
+      "step": 564
+    },
+    {
+      "epoch": 0.7031736154324829,
+      "grad_norm": 2.827235460281372,
+      "learning_rate": 4.287361623616236e-06,
+      "loss": 1.5333,
+      "step": 565
+    },
+    {
+      "epoch": 0.7044181705040448,
+      "grad_norm": 2.6359963417053223,
+      "learning_rate": 4.285055350553506e-06,
+      "loss": 1.5849,
+      "step": 566
+    },
+    {
+      "epoch": 0.7056627255756067,
+      "grad_norm": 2.930530071258545,
+      "learning_rate": 4.282749077490776e-06,
+      "loss": 1.5672,
+      "step": 567
+    },
+    {
+      "epoch": 0.7069072806471687,
+      "grad_norm": 2.750102996826172,
+      "learning_rate": 4.280442804428044e-06,
+      "loss": 1.5696,
+      "step": 568
+    },
+    {
+      "epoch": 0.7081518357187305,
+      "grad_norm": 2.869690418243408,
+      "learning_rate": 4.278136531365314e-06,
+      "loss": 1.5689,
+      "step": 569
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "grad_norm": 2.954852819442749,
+      "learning_rate": 4.275830258302583e-06,
+      "loss": 1.5931,
+      "step": 570
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "eval_loss": 1.7143300771713257,
+      "eval_runtime": 52.7293,
+      "eval_samples_per_second": 18.965,
+      "eval_steps_per_second": 0.797,
+      "step": 570
+    },
+    {
+      "epoch": 0.7106409458618543,
+      "grad_norm": 2.702223539352417,
+      "learning_rate": 4.273523985239853e-06,
+      "loss": 1.5635,
+      "step": 571
+    },
+    {
+      "epoch": 0.7118855009334163,
+      "grad_norm": 2.689995050430298,
+      "learning_rate": 4.271217712177122e-06,
+      "loss": 1.5545,
+      "step": 572
+    },
+    {
+      "epoch": 0.7131300560049783,
+      "grad_norm": 2.68979811668396,
+      "learning_rate": 4.268911439114391e-06,
+      "loss": 1.5404,
+      "step": 573
+    },
+    {
+      "epoch": 0.7143746110765401,
+      "grad_norm": 2.7477986812591553,
+      "learning_rate": 4.266605166051661e-06,
+      "loss": 1.5719,
+      "step": 574
+    },
+    {
+      "epoch": 0.7156191661481021,
+      "grad_norm": 2.975778341293335,
+      "learning_rate": 4.26429889298893e-06,
+      "loss": 1.5587,
+      "step": 575
+    },
+    {
+      "epoch": 0.716863721219664,
+      "grad_norm": 2.658170700073242,
+      "learning_rate": 4.2619926199262e-06,
+      "loss": 1.5746,
+      "step": 576
+    },
+    {
+      "epoch": 0.7181082762912259,
+      "grad_norm": 2.8825011253356934,
+      "learning_rate": 4.259686346863469e-06,
+      "loss": 1.5246,
+      "step": 577
+    },
+    {
+      "epoch": 0.7193528313627878,
+      "grad_norm": 2.845280170440674,
+      "learning_rate": 4.2573800738007385e-06,
+      "loss": 1.5703,
+      "step": 578
+    },
+    {
+      "epoch": 0.7205973864343497,
+      "grad_norm": 2.660616636276245,
+      "learning_rate": 4.255073800738008e-06,
+      "loss": 1.5373,
+      "step": 579
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "grad_norm": 2.749447822570801,
+      "learning_rate": 4.252767527675277e-06,
+      "loss": 1.59,
+      "step": 580
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "eval_loss": 1.7149444818496704,
+      "eval_runtime": 52.2099,
+      "eval_samples_per_second": 19.153,
+      "eval_steps_per_second": 0.804,
+      "step": 580
+    },
+    {
+      "epoch": 0.7230864965774736,
+      "grad_norm": 2.813328742980957,
+      "learning_rate": 4.250461254612546e-06,
+      "loss": 1.5694,
+      "step": 581
+    },
+    {
+      "epoch": 0.7243310516490354,
+      "grad_norm": 2.823866844177246,
+      "learning_rate": 4.248154981549816e-06,
+      "loss": 1.5143,
+      "step": 582
+    },
+    {
+      "epoch": 0.7255756067205974,
+      "grad_norm": 2.71337890625,
+      "learning_rate": 4.245848708487086e-06,
+      "loss": 1.5915,
+      "step": 583
+    },
+    {
+      "epoch": 0.7268201617921594,
+      "grad_norm": 2.718085765838623,
+      "learning_rate": 4.243542435424355e-06,
+      "loss": 1.604,
+      "step": 584
+    },
+    {
+      "epoch": 0.7280647168637212,
+      "grad_norm": 2.5741796493530273,
+      "learning_rate": 4.241236162361624e-06,
+      "loss": 1.5398,
+      "step": 585
+    },
+    {
+      "epoch": 0.7293092719352832,
+      "grad_norm": 2.727114200592041,
+      "learning_rate": 4.238929889298893e-06,
+      "loss": 1.5886,
+      "step": 586
+    },
+    {
+      "epoch": 0.730553827006845,
+      "grad_norm": 2.6703338623046875,
+      "learning_rate": 4.236623616236163e-06,
+      "loss": 1.5959,
+      "step": 587
+    },
+    {
+      "epoch": 0.731798382078407,
+      "grad_norm": 2.750326633453369,
+      "learning_rate": 4.234317343173432e-06,
+      "loss": 1.5916,
+      "step": 588
+    },
+    {
+      "epoch": 0.7330429371499689,
+      "grad_norm": 2.6680715084075928,
+      "learning_rate": 4.232011070110701e-06,
+      "loss": 1.562,
+      "step": 589
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "grad_norm": 2.6603832244873047,
+      "learning_rate": 4.229704797047971e-06,
+      "loss": 1.6121,
+      "step": 590
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "eval_loss": 1.7144734859466553,
+      "eval_runtime": 50.5317,
+      "eval_samples_per_second": 19.79,
+      "eval_steps_per_second": 0.831,
+      "step": 590
+    },
+    {
+      "epoch": 0.7355320472930927,
+      "grad_norm": 2.7868523597717285,
+      "learning_rate": 4.22739852398524e-06,
+      "loss": 1.5488,
+      "step": 591
+    },
+    {
+      "epoch": 0.7367766023646546,
+      "grad_norm": 2.755270004272461,
+      "learning_rate": 4.2250922509225096e-06,
+      "loss": 1.5801,
+      "step": 592
+    },
+    {
+      "epoch": 0.7380211574362165,
+      "grad_norm": 2.7614126205444336,
+      "learning_rate": 4.222785977859779e-06,
+      "loss": 1.5507,
+      "step": 593
+    },
+    {
+      "epoch": 0.7392657125077785,
+      "grad_norm": 2.8674862384796143,
+      "learning_rate": 4.220479704797048e-06,
+      "loss": 1.5807,
+      "step": 594
+    },
+    {
+      "epoch": 0.7405102675793404,
+      "grad_norm": 2.7366859912872314,
+      "learning_rate": 4.218173431734318e-06,
+      "loss": 1.6594,
+      "step": 595
+    },
+    {
+      "epoch": 0.7417548226509023,
+      "grad_norm": 2.6231045722961426,
+      "learning_rate": 4.215867158671587e-06,
+      "loss": 1.5609,
+      "step": 596
+    },
+    {
+      "epoch": 0.7429993777224643,
+      "grad_norm": 2.7694146633148193,
+      "learning_rate": 4.213560885608857e-06,
+      "loss": 1.5889,
+      "step": 597
+    },
+    {
+      "epoch": 0.7442439327940261,
+      "grad_norm": 2.6275386810302734,
+      "learning_rate": 4.211254612546125e-06,
+      "loss": 1.575,
+      "step": 598
+    },
+    {
+      "epoch": 0.7454884878655881,
+      "grad_norm": 2.9413866996765137,
+      "learning_rate": 4.2089483394833955e-06,
+      "loss": 1.589,
+      "step": 599
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "grad_norm": 2.768606185913086,
+      "learning_rate": 4.206642066420665e-06,
+      "loss": 1.5738,
+      "step": 600
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "eval_loss": 1.7148027420043945,
+      "eval_runtime": 54.7031,
+      "eval_samples_per_second": 18.28,
+      "eval_steps_per_second": 0.768,
+      "step": 600
+    },
+    {
+      "epoch": 0.7479775980087119,
+      "grad_norm": 2.7917706966400146,
+      "learning_rate": 4.2043357933579335e-06,
+      "loss": 1.499,
+      "step": 601
+    },
+    {
+      "epoch": 0.7492221530802738,
+      "grad_norm": 2.5982208251953125,
+      "learning_rate": 4.202029520295203e-06,
+      "loss": 1.5113,
+      "step": 602
+    },
+    {
+      "epoch": 0.7504667081518357,
+      "grad_norm": 2.692032814025879,
+      "learning_rate": 4.199723247232473e-06,
+      "loss": 1.4721,
+      "step": 603
+    },
+    {
+      "epoch": 0.7517112632233977,
+      "grad_norm": 2.975860834121704,
+      "learning_rate": 4.197416974169742e-06,
+      "loss": 1.5147,
+      "step": 604
+    },
+    {
+      "epoch": 0.7529558182949595,
+      "grad_norm": 2.758610248565674,
+      "learning_rate": 4.195110701107011e-06,
+      "loss": 1.5674,
+      "step": 605
+    },
+    {
+      "epoch": 0.7542003733665215,
+      "grad_norm": 2.7620184421539307,
+      "learning_rate": 4.192804428044281e-06,
+      "loss": 1.5669,
+      "step": 606
+    },
+    {
+      "epoch": 0.7554449284380834,
+      "grad_norm": 3.043940305709839,
+      "learning_rate": 4.19049815498155e-06,
+      "loss": 1.587,
+      "step": 607
+    },
+    {
+      "epoch": 0.7566894835096453,
+      "grad_norm": 2.7482151985168457,
+      "learning_rate": 4.1881918819188195e-06,
+      "loss": 1.5451,
+      "step": 608
+    },
+    {
+      "epoch": 0.7579340385812072,
+      "grad_norm": 2.7259294986724854,
+      "learning_rate": 4.185885608856089e-06,
+      "loss": 1.5295,
+      "step": 609
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "grad_norm": 2.6340832710266113,
+      "learning_rate": 4.183579335793358e-06,
+      "loss": 1.5335,
+      "step": 610
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "eval_loss": 1.7038393020629883,
+      "eval_runtime": 44.0792,
+      "eval_samples_per_second": 22.686,
+      "eval_steps_per_second": 0.953,
+      "step": 610
+    },
+    {
+      "epoch": 0.760423148724331,
+      "grad_norm": 2.6144909858703613,
+      "learning_rate": 4.181273062730628e-06,
+      "loss": 1.5907,
+      "step": 611
+    },
+    {
+      "epoch": 0.761667703795893,
+      "grad_norm": 2.7754175662994385,
+      "learning_rate": 4.178966789667897e-06,
+      "loss": 1.5773,
+      "step": 612
+    },
+    {
+      "epoch": 0.7629122588674548,
+      "grad_norm": 2.6313252449035645,
+      "learning_rate": 4.176660516605167e-06,
+      "loss": 1.5675,
+      "step": 613
+    },
+    {
+      "epoch": 0.7641568139390168,
+      "grad_norm": 2.549074649810791,
+      "learning_rate": 4.174354243542435e-06,
+      "loss": 1.595,
+      "step": 614
+    },
+    {
+      "epoch": 0.7654013690105788,
+      "grad_norm": 2.611804246902466,
+      "learning_rate": 4.1720479704797054e-06,
+      "loss": 1.5416,
+      "step": 615
+    },
+    {
+      "epoch": 0.7666459240821406,
+      "grad_norm": 2.6322927474975586,
+      "learning_rate": 4.169741697416975e-06,
+      "loss": 1.604,
+      "step": 616
+    },
+    {
+      "epoch": 0.7678904791537026,
+      "grad_norm": 2.5792219638824463,
+      "learning_rate": 4.1674354243542434e-06,
+      "loss": 1.5976,
+      "step": 617
+    },
+    {
+      "epoch": 0.7691350342252644,
+      "grad_norm": 2.845416307449341,
+      "learning_rate": 4.165129151291513e-06,
+      "loss": 1.5945,
+      "step": 618
+    },
+    {
+      "epoch": 0.7703795892968264,
+      "grad_norm": 2.8647871017456055,
+      "learning_rate": 4.162822878228783e-06,
+      "loss": 1.5637,
+      "step": 619
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "grad_norm": 2.590719699859619,
+      "learning_rate": 4.1605166051660526e-06,
+      "loss": 1.556,
+      "step": 620
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "eval_loss": 1.7115222215652466,
+      "eval_runtime": 50.3905,
+      "eval_samples_per_second": 19.845,
+      "eval_steps_per_second": 0.833,
+      "step": 620
+    },
+    {
+      "epoch": 0.7728686994399502,
+      "grad_norm": 2.626723527908325,
+      "learning_rate": 4.158210332103321e-06,
+      "loss": 1.5486,
+      "step": 621
+    },
+    {
+      "epoch": 0.7741132545115121,
+      "grad_norm": 2.713069438934326,
+      "learning_rate": 4.1559040590405906e-06,
+      "loss": 1.5474,
+      "step": 622
+    },
+    {
+      "epoch": 0.7753578095830741,
+      "grad_norm": 2.685563087463379,
+      "learning_rate": 4.15359778597786e-06,
+      "loss": 1.5529,
+      "step": 623
+    },
+    {
+      "epoch": 0.776602364654636,
+      "grad_norm": 2.690586805343628,
+      "learning_rate": 4.151291512915129e-06,
+      "loss": 1.5293,
+      "step": 624
+    },
+    {
+      "epoch": 0.7778469197261979,
+      "grad_norm": 2.865345001220703,
+      "learning_rate": 4.148985239852399e-06,
+      "loss": 1.5095,
+      "step": 625
+    },
+    {
+      "epoch": 0.7790914747977598,
+      "grad_norm": 2.92393159866333,
+      "learning_rate": 4.146678966789668e-06,
+      "loss": 1.5561,
+      "step": 626
+    },
+    {
+      "epoch": 0.7803360298693217,
+      "grad_norm": 2.554480791091919,
+      "learning_rate": 4.144372693726938e-06,
+      "loss": 1.4876,
+      "step": 627
+    },
+    {
+      "epoch": 0.7815805849408837,
+      "grad_norm": 2.6128833293914795,
+      "learning_rate": 4.142066420664207e-06,
+      "loss": 1.5378,
+      "step": 628
+    },
+    {
+      "epoch": 0.7828251400124455,
+      "grad_norm": 3.0110888481140137,
+      "learning_rate": 4.1397601476014765e-06,
+      "loss": 1.5859,
+      "step": 629
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "grad_norm": 2.8152706623077393,
+      "learning_rate": 4.137453874538745e-06,
+      "loss": 1.5841,
+      "step": 630
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "eval_loss": 1.702181339263916,
+      "eval_runtime": 53.6926,
+      "eval_samples_per_second": 18.625,
+      "eval_steps_per_second": 0.782,
+      "step": 630
+    },
+    {
+      "epoch": 0.7853142501555694,
+      "grad_norm": 2.8634233474731445,
+      "learning_rate": 4.135147601476015e-06,
+      "loss": 1.564,
+      "step": 631
+    },
+    {
+      "epoch": 0.7865588052271313,
+      "grad_norm": 2.8939666748046875,
+      "learning_rate": 4.132841328413285e-06,
+      "loss": 1.5859,
+      "step": 632
+    },
+    {
+      "epoch": 0.7878033602986932,
+      "grad_norm": 2.6560444831848145,
+      "learning_rate": 4.130535055350554e-06,
+      "loss": 1.5199,
+      "step": 633
+    },
+    {
+      "epoch": 0.7890479153702551,
+      "grad_norm": 2.964721202850342,
+      "learning_rate": 4.128228782287823e-06,
+      "loss": 1.5888,
+      "step": 634
+    },
+    {
+      "epoch": 0.790292470441817,
+      "grad_norm": 2.74668288230896,
+      "learning_rate": 4.125922509225092e-06,
+      "loss": 1.5847,
+      "step": 635
+    },
+    {
+      "epoch": 0.791537025513379,
+      "grad_norm": 2.723123550415039,
+      "learning_rate": 4.1236162361623625e-06,
+      "loss": 1.5166,
+      "step": 636
+    },
+    {
+      "epoch": 0.7927815805849409,
+      "grad_norm": 2.7265713214874268,
+      "learning_rate": 4.121309963099631e-06,
+      "loss": 1.5954,
+      "step": 637
+    },
+    {
+      "epoch": 0.7940261356565028,
+      "grad_norm": 2.979126214981079,
+      "learning_rate": 4.1190036900369005e-06,
+      "loss": 1.5583,
+      "step": 638
+    },
+    {
+      "epoch": 0.7952706907280647,
+      "grad_norm": 2.844376802444458,
+      "learning_rate": 4.11669741697417e-06,
+      "loss": 1.5564,
+      "step": 639
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "grad_norm": 2.7583703994750977,
+      "learning_rate": 4.114391143911439e-06,
+      "loss": 1.5094,
+      "step": 640
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "eval_loss": 1.6986565589904785,
+      "eval_runtime": 47.3549,
+      "eval_samples_per_second": 21.117,
+      "eval_steps_per_second": 0.887,
+      "step": 640
+    },
+    {
+      "epoch": 0.7977598008711886,
+      "grad_norm": 2.8942224979400635,
+      "learning_rate": 4.112084870848709e-06,
+      "loss": 1.5504,
+      "step": 641
+    },
+    {
+      "epoch": 0.7990043559427504,
+      "grad_norm": 2.6590495109558105,
+      "learning_rate": 4.109778597785978e-06,
+      "loss": 1.5227,
+      "step": 642
+    },
+    {
+      "epoch": 0.8002489110143124,
+      "grad_norm": 2.5988378524780273,
+      "learning_rate": 4.107472324723248e-06,
+      "loss": 1.5054,
+      "step": 643
+    },
+    {
+      "epoch": 0.8014934660858744,
+      "grad_norm": 2.787335157394409,
+      "learning_rate": 4.105166051660517e-06,
+      "loss": 1.5844,
+      "step": 644
+    },
+    {
+      "epoch": 0.8027380211574362,
+      "grad_norm": 2.8202896118164062,
+      "learning_rate": 4.1028597785977864e-06,
+      "loss": 1.5596,
+      "step": 645
+    },
+    {
+      "epoch": 0.8039825762289982,
+      "grad_norm": 2.65376615524292,
+      "learning_rate": 4.100553505535056e-06,
+      "loss": 1.5893,
+      "step": 646
+    },
+    {
+      "epoch": 0.80522713130056,
+      "grad_norm": 2.5933308601379395,
+      "learning_rate": 4.098247232472325e-06,
+      "loss": 1.5403,
+      "step": 647
+    },
+    {
+      "epoch": 0.806471686372122,
+      "grad_norm": 2.777070999145508,
+      "learning_rate": 4.095940959409595e-06,
+      "loss": 1.5231,
+      "step": 648
+    },
+    {
+      "epoch": 0.8077162414436839,
+      "grad_norm": 2.6427664756774902,
+      "learning_rate": 4.093634686346864e-06,
+      "loss": 1.5937,
+      "step": 649
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "grad_norm": 2.651561737060547,
+      "learning_rate": 4.091328413284133e-06,
+      "loss": 1.4847,
+      "step": 650
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "eval_loss": 1.7089996337890625,
+      "eval_runtime": 47.4346,
+      "eval_samples_per_second": 21.082,
+      "eval_steps_per_second": 0.885,
+      "step": 650
+    },
+    {
+      "epoch": 0.8102053515868077,
+      "grad_norm": 2.844067335128784,
+      "learning_rate": 4.089022140221402e-06,
+      "loss": 1.488,
+      "step": 651
+    },
+    {
+      "epoch": 0.8114499066583696,
+      "grad_norm": 2.603480577468872,
+      "learning_rate": 4.086715867158672e-06,
+      "loss": 1.5248,
+      "step": 652
+    },
+    {
+      "epoch": 0.8126944617299315,
+      "grad_norm": 2.789322853088379,
+      "learning_rate": 4.084409594095941e-06,
+      "loss": 1.5119,
+      "step": 653
+    },
+    {
+      "epoch": 0.8139390168014935,
+      "grad_norm": 2.7183773517608643,
+      "learning_rate": 4.08210332103321e-06,
+      "loss": 1.5123,
+      "step": 654
+    },
+    {
+      "epoch": 0.8151835718730553,
+      "grad_norm": 2.80613374710083,
+      "learning_rate": 4.07979704797048e-06,
+      "loss": 1.5526,
+      "step": 655
+    },
+    {
+      "epoch": 0.8164281269446173,
+      "grad_norm": 2.641035795211792,
+      "learning_rate": 4.077490774907749e-06,
+      "loss": 1.6127,
+      "step": 656
+    },
+    {
+      "epoch": 0.8176726820161793,
+      "grad_norm": 2.773684024810791,
+      "learning_rate": 4.075184501845019e-06,
+      "loss": 1.5404,
+      "step": 657
+    },
+    {
+      "epoch": 0.8189172370877411,
+      "grad_norm": 2.6951382160186768,
+      "learning_rate": 4.072878228782288e-06,
+      "loss": 1.4928,
+      "step": 658
+    },
+    {
+      "epoch": 0.8201617921593031,
+      "grad_norm": 2.931771755218506,
+      "learning_rate": 4.0705719557195575e-06,
+      "loss": 1.5949,
+      "step": 659
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "grad_norm": 2.8786075115203857,
+      "learning_rate": 4.068265682656827e-06,
+      "loss": 1.5582,
+      "step": 660
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "eval_loss": 1.698564052581787,
+      "eval_runtime": 46.6299,
+      "eval_samples_per_second": 21.445,
+      "eval_steps_per_second": 0.901,
+      "step": 660
+    },
+    {
+      "epoch": 0.8226509023024269,
+      "grad_norm": 2.7402451038360596,
+      "learning_rate": 4.065959409594096e-06,
+      "loss": 1.5562,
+      "step": 661
+    },
+    {
+      "epoch": 0.8238954573739888,
+      "grad_norm": 2.7968692779541016,
+      "learning_rate": 4.063653136531366e-06,
+      "loss": 1.556,
+      "step": 662
+    },
+    {
+      "epoch": 0.8251400124455507,
+      "grad_norm": 2.8233132362365723,
+      "learning_rate": 4.061346863468635e-06,
+      "loss": 1.5527,
+      "step": 663
+    },
+    {
+      "epoch": 0.8263845675171126,
+      "grad_norm": 2.713390827178955,
+      "learning_rate": 4.059040590405905e-06,
+      "loss": 1.6012,
+      "step": 664
+    },
+    {
+      "epoch": 0.8276291225886746,
+      "grad_norm": 2.7783989906311035,
+      "learning_rate": 4.056734317343174e-06,
+      "loss": 1.5247,
+      "step": 665
+    },
+    {
+      "epoch": 0.8288736776602365,
+      "grad_norm": 2.808554172515869,
+      "learning_rate": 4.054428044280443e-06,
+      "loss": 1.5412,
+      "step": 666
+    },
+    {
+      "epoch": 0.8301182327317984,
+      "grad_norm": 2.8476831912994385,
+      "learning_rate": 4.052121771217712e-06,
+      "loss": 1.5307,
+      "step": 667
+    },
+    {
+      "epoch": 0.8313627878033603,
+      "grad_norm": 2.8287601470947266,
+      "learning_rate": 4.049815498154982e-06,
+      "loss": 1.6021,
+      "step": 668
+    },
+    {
+      "epoch": 0.8326073428749222,
+      "grad_norm": 2.975510835647583,
+      "learning_rate": 4.047509225092252e-06,
+      "loss": 1.5474,
+      "step": 669
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "grad_norm": 2.723236560821533,
+      "learning_rate": 4.04520295202952e-06,
+      "loss": 1.5242,
+      "step": 670
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "eval_loss": 1.7005239725112915,
+      "eval_runtime": 49.4504,
+      "eval_samples_per_second": 20.222,
+      "eval_steps_per_second": 0.849,
+      "step": 670
+    },
+    {
+      "epoch": 0.835096453018046,
+      "grad_norm": 2.7521748542785645,
+      "learning_rate": 4.04289667896679e-06,
+      "loss": 1.5467,
+      "step": 671
+    },
+    {
+      "epoch": 0.836341008089608,
+      "grad_norm": 2.7770044803619385,
+      "learning_rate": 4.04059040590406e-06,
+      "loss": 1.5407,
+      "step": 672
+    },
+    {
+      "epoch": 0.8375855631611698,
+      "grad_norm": 2.744323492050171,
+      "learning_rate": 4.038284132841329e-06,
+      "loss": 1.5422,
+      "step": 673
+    },
+    {
+      "epoch": 0.8388301182327318,
+      "grad_norm": 2.6699817180633545,
+      "learning_rate": 4.035977859778598e-06,
+      "loss": 1.5508,
+      "step": 674
+    },
+    {
+      "epoch": 0.8400746733042938,
+      "grad_norm": 2.705273151397705,
+      "learning_rate": 4.0336715867158674e-06,
+      "loss": 1.5099,
+      "step": 675
+    },
+    {
+      "epoch": 0.8413192283758556,
+      "grad_norm": 2.9485747814178467,
+      "learning_rate": 4.031365313653137e-06,
+      "loss": 1.5529,
+      "step": 676
+    },
+    {
+      "epoch": 0.8425637834474176,
+      "grad_norm": 3.1750423908233643,
+      "learning_rate": 4.029059040590406e-06,
+      "loss": 1.5894,
+      "step": 677
+    },
+    {
+      "epoch": 0.8438083385189795,
+      "grad_norm": 2.76448655128479,
+      "learning_rate": 4.026752767527676e-06,
+      "loss": 1.5752,
+      "step": 678
+    },
+    {
+      "epoch": 0.8450528935905414,
+      "grad_norm": 2.676708698272705,
+      "learning_rate": 4.024446494464945e-06,
+      "loss": 1.5487,
+      "step": 679
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "grad_norm": 2.6832051277160645,
+      "learning_rate": 4.0221402214022145e-06,
+      "loss": 1.5055,
+      "step": 680
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "eval_loss": 1.6945017576217651,
+      "eval_runtime": 43.9369,
+      "eval_samples_per_second": 22.76,
+      "eval_steps_per_second": 0.956,
+      "step": 680
+    },
+    {
+      "epoch": 0.8475420037336652,
+      "grad_norm": 2.6812336444854736,
+      "learning_rate": 4.019833948339484e-06,
+      "loss": 1.5315,
+      "step": 681
+    },
+    {
+      "epoch": 0.8487865588052271,
+      "grad_norm": 2.9394326210021973,
+      "learning_rate": 4.017527675276753e-06,
+      "loss": 1.5636,
+      "step": 682
+    },
+    {
+      "epoch": 0.8500311138767891,
+      "grad_norm": 2.944952964782715,
+      "learning_rate": 4.015221402214022e-06,
+      "loss": 1.56,
+      "step": 683
+    },
+    {
+      "epoch": 0.8512756689483509,
+      "grad_norm": 2.7475314140319824,
+      "learning_rate": 4.012915129151292e-06,
+      "loss": 1.5442,
+      "step": 684
+    },
+    {
+      "epoch": 0.8525202240199129,
+      "grad_norm": 3.0572879314422607,
+      "learning_rate": 4.010608856088562e-06,
+      "loss": 1.5023,
+      "step": 685
+    },
+    {
+      "epoch": 0.8537647790914747,
+      "grad_norm": 2.75365948677063,
+      "learning_rate": 4.00830258302583e-06,
+      "loss": 1.5414,
+      "step": 686
+    },
+    {
+      "epoch": 0.8550093341630367,
+      "grad_norm": 2.5714449882507324,
+      "learning_rate": 4.0059963099631e-06,
+      "loss": 1.5027,
+      "step": 687
+    },
+    {
+      "epoch": 0.8562538892345987,
+      "grad_norm": 2.755167007446289,
+      "learning_rate": 4.003690036900369e-06,
+      "loss": 1.5788,
+      "step": 688
+    },
+    {
+      "epoch": 0.8574984443061605,
+      "grad_norm": 2.798967123031616,
+      "learning_rate": 4.0013837638376385e-06,
+      "loss": 1.5162,
+      "step": 689
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "grad_norm": 2.803614854812622,
+      "learning_rate": 3.999077490774908e-06,
+      "loss": 1.5079,
+      "step": 690
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "eval_loss": 1.6924962997436523,
+      "eval_runtime": 44.47,
+      "eval_samples_per_second": 22.487,
+      "eval_steps_per_second": 0.944,
+      "step": 690
+    },
+    {
+      "epoch": 0.8599875544492844,
+      "grad_norm": 2.6898066997528076,
+      "learning_rate": 3.996771217712177e-06,
+      "loss": 1.5478,
+      "step": 691
+    },
+    {
+      "epoch": 0.8612321095208463,
+      "grad_norm": 2.6376144886016846,
+      "learning_rate": 3.994464944649447e-06,
+      "loss": 1.5362,
+      "step": 692
+    },
+    {
+      "epoch": 0.8624766645924082,
+      "grad_norm": 2.7240405082702637,
+      "learning_rate": 3.992158671586716e-06,
+      "loss": 1.5502,
+      "step": 693
+    },
+    {
+      "epoch": 0.8637212196639701,
+      "grad_norm": 2.929445266723633,
+      "learning_rate": 3.989852398523986e-06,
+      "loss": 1.5799,
+      "step": 694
+    },
+    {
+      "epoch": 0.864965774735532,
+      "grad_norm": 2.593223810195923,
+      "learning_rate": 3.987546125461255e-06,
+      "loss": 1.5352,
+      "step": 695
+    },
+    {
+      "epoch": 0.866210329807094,
+      "grad_norm": 2.7710154056549072,
+      "learning_rate": 3.9852398523985245e-06,
+      "loss": 1.5048,
+      "step": 696
+    },
+    {
+      "epoch": 0.8674548848786559,
+      "grad_norm": 2.7370848655700684,
+      "learning_rate": 3.982933579335794e-06,
+      "loss": 1.5233,
+      "step": 697
+    },
+    {
+      "epoch": 0.8686994399502178,
+      "grad_norm": 2.8632168769836426,
+      "learning_rate": 3.980627306273063e-06,
+      "loss": 1.5435,
+      "step": 698
+    },
+    {
+      "epoch": 0.8699439950217797,
+      "grad_norm": 2.752298593521118,
+      "learning_rate": 3.978321033210332e-06,
+      "loss": 1.5728,
+      "step": 699
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "grad_norm": 2.8345625400543213,
+      "learning_rate": 3.976014760147602e-06,
+      "loss": 1.5727,
+      "step": 700
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "eval_loss": 1.6855305433273315,
+      "eval_runtime": 43.3683,
+      "eval_samples_per_second": 23.058,
+      "eval_steps_per_second": 0.968,
+      "step": 700
+    },
+    {
+      "epoch": 0.8724331051649036,
+      "grad_norm": 2.8135335445404053,
+      "learning_rate": 3.973708487084872e-06,
+      "loss": 1.5086,
+      "step": 701
+    },
+    {
+      "epoch": 0.8736776602364654,
+      "grad_norm": 2.7416603565216064,
+      "learning_rate": 3.97140221402214e-06,
+      "loss": 1.545,
+      "step": 702
+    },
+    {
+      "epoch": 0.8749222153080274,
+      "grad_norm": 2.6902289390563965,
+      "learning_rate": 3.96909594095941e-06,
+      "loss": 1.5509,
+      "step": 703
+    },
+    {
+      "epoch": 0.8761667703795893,
+      "grad_norm": 2.6537084579467773,
+      "learning_rate": 3.966789667896679e-06,
+      "loss": 1.562,
+      "step": 704
+    },
+    {
+      "epoch": 0.8774113254511512,
+      "grad_norm": 2.6979284286499023,
+      "learning_rate": 3.964483394833948e-06,
+      "loss": 1.4829,
+      "step": 705
+    },
+    {
+      "epoch": 0.8786558805227132,
+      "grad_norm": 2.5795764923095703,
+      "learning_rate": 3.962177121771218e-06,
+      "loss": 1.4677,
+      "step": 706
+    },
+    {
+      "epoch": 0.879900435594275,
+      "grad_norm": 3.0185086727142334,
+      "learning_rate": 3.959870848708487e-06,
+      "loss": 1.5608,
+      "step": 707
+    },
+    {
+      "epoch": 0.881144990665837,
+      "grad_norm": 2.6458113193511963,
+      "learning_rate": 3.957564575645757e-06,
+      "loss": 1.5342,
+      "step": 708
+    },
+    {
+      "epoch": 0.8823895457373989,
+      "grad_norm": 2.7985665798187256,
+      "learning_rate": 3.955258302583026e-06,
+      "loss": 1.527,
+      "step": 709
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "grad_norm": 2.9069082736968994,
+      "learning_rate": 3.9529520295202955e-06,
+      "loss": 1.5156,
+      "step": 710
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "eval_loss": 1.6898822784423828,
+      "eval_runtime": 42.8749,
+      "eval_samples_per_second": 23.324,
+      "eval_steps_per_second": 0.98,
+      "step": 710
+    },
+    {
+      "epoch": 0.8848786558805227,
+      "grad_norm": 2.760143995285034,
+      "learning_rate": 3.950645756457565e-06,
+      "loss": 1.5124,
+      "step": 711
+    },
+    {
+      "epoch": 0.8861232109520847,
+      "grad_norm": 2.8085529804229736,
+      "learning_rate": 3.948339483394834e-06,
+      "loss": 1.5862,
+      "step": 712
+    },
+    {
+      "epoch": 0.8873677660236465,
+      "grad_norm": 2.909905195236206,
+      "learning_rate": 3.946033210332104e-06,
+      "loss": 1.5557,
+      "step": 713
+    },
+    {
+      "epoch": 0.8886123210952085,
+      "grad_norm": 2.826899290084839,
+      "learning_rate": 3.943726937269373e-06,
+      "loss": 1.5277,
+      "step": 714
+    },
+    {
+      "epoch": 0.8898568761667703,
+      "grad_norm": 2.6269052028656006,
+      "learning_rate": 3.941420664206642e-06,
+      "loss": 1.4819,
+      "step": 715
+    },
+    {
+      "epoch": 0.8911014312383323,
+      "grad_norm": 2.6687698364257812,
+      "learning_rate": 3.939114391143912e-06,
+      "loss": 1.5156,
+      "step": 716
+    },
+    {
+      "epoch": 0.8923459863098943,
+      "grad_norm": 2.9605629444122314,
+      "learning_rate": 3.9368081180811815e-06,
+      "loss": 1.5358,
+      "step": 717
+    },
+    {
+      "epoch": 0.8935905413814561,
+      "grad_norm": 3.0367166996002197,
+      "learning_rate": 3.934501845018451e-06,
+      "loss": 1.5847,
+      "step": 718
+    },
+    {
+      "epoch": 0.8948350964530181,
+      "grad_norm": 2.528796434402466,
+      "learning_rate": 3.9321955719557195e-06,
+      "loss": 1.4851,
+      "step": 719
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "grad_norm": 2.7116613388061523,
+      "learning_rate": 3.929889298892989e-06,
+      "loss": 1.4706,
+      "step": 720
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "eval_loss": 1.6845883131027222,
+      "eval_runtime": 44.4734,
+      "eval_samples_per_second": 22.485,
+      "eval_steps_per_second": 0.944,
+      "step": 720
+    },
+    {
+      "epoch": 0.8973242065961419,
+      "grad_norm": 2.771864414215088,
+      "learning_rate": 3.927583025830259e-06,
+      "loss": 1.5444,
+      "step": 721
+    },
+    {
+      "epoch": 0.8985687616677038,
+      "grad_norm": 2.773746967315674,
+      "learning_rate": 3.925276752767528e-06,
+      "loss": 1.564,
+      "step": 722
+    },
+    {
+      "epoch": 0.8998133167392657,
+      "grad_norm": 2.7069854736328125,
+      "learning_rate": 3.922970479704797e-06,
+      "loss": 1.5601,
+      "step": 723
+    },
+    {
+      "epoch": 0.9010578718108276,
+      "grad_norm": 2.840261697769165,
+      "learning_rate": 3.920664206642067e-06,
+      "loss": 1.5733,
+      "step": 724
+    },
+    {
+      "epoch": 0.9023024268823896,
+      "grad_norm": 2.709897041320801,
+      "learning_rate": 3.918357933579336e-06,
+      "loss": 1.528,
+      "step": 725
+    },
+    {
+      "epoch": 0.9035469819539514,
+      "grad_norm": 2.662367343902588,
+      "learning_rate": 3.9160516605166055e-06,
+      "loss": 1.5532,
+      "step": 726
+    },
+    {
+      "epoch": 0.9047915370255134,
+      "grad_norm": 2.6775717735290527,
+      "learning_rate": 3.913745387453875e-06,
+      "loss": 1.5464,
+      "step": 727
+    },
+    {
+      "epoch": 0.9060360920970753,
+      "grad_norm": 2.617842674255371,
+      "learning_rate": 3.911439114391144e-06,
+      "loss": 1.4539,
+      "step": 728
+    },
+    {
+      "epoch": 0.9072806471686372,
+      "grad_norm": 2.668649435043335,
+      "learning_rate": 3.909132841328414e-06,
+      "loss": 1.5627,
+      "step": 729
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "grad_norm": 2.5331950187683105,
+      "learning_rate": 3.906826568265683e-06,
+      "loss": 1.5165,
+      "step": 730
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "eval_loss": 1.6860331296920776,
+      "eval_runtime": 46.094,
+      "eval_samples_per_second": 21.695,
+      "eval_steps_per_second": 0.911,
+      "step": 730
+    },
+    {
+      "epoch": 0.909769757311761,
+      "grad_norm": 2.602492332458496,
+      "learning_rate": 3.9045202952029526e-06,
+      "loss": 1.5145,
+      "step": 731
+    },
+    {
+      "epoch": 0.911014312383323,
+      "grad_norm": 2.632782459259033,
+      "learning_rate": 3.902214022140222e-06,
+      "loss": 1.4413,
+      "step": 732
+    },
+    {
+      "epoch": 0.9122588674548848,
+      "grad_norm": 2.6025912761688232,
+      "learning_rate": 3.899907749077491e-06,
+      "loss": 1.5263,
+      "step": 733
+    },
+    {
+      "epoch": 0.9135034225264468,
+      "grad_norm": 2.770116090774536,
+      "learning_rate": 3.897601476014761e-06,
+      "loss": 1.5514,
+      "step": 734
+    },
+    {
+      "epoch": 0.9147479775980087,
+      "grad_norm": 2.7822961807250977,
+      "learning_rate": 3.895295202952029e-06,
+      "loss": 1.5296,
+      "step": 735
+    },
+    {
+      "epoch": 0.9159925326695706,
+      "grad_norm": 2.6689720153808594,
+      "learning_rate": 3.892988929889299e-06,
+      "loss": 1.4552,
+      "step": 736
+    },
+    {
+      "epoch": 0.9172370877411326,
+      "grad_norm": 2.8021798133850098,
+      "learning_rate": 3.890682656826569e-06,
+      "loss": 1.5308,
+      "step": 737
+    },
+    {
+      "epoch": 0.9184816428126945,
+      "grad_norm": 2.575313091278076,
+      "learning_rate": 3.888376383763838e-06,
+      "loss": 1.4807,
+      "step": 738
+    },
+    {
+      "epoch": 0.9197261978842564,
+      "grad_norm": 2.5955779552459717,
+      "learning_rate": 3.886070110701107e-06,
+      "loss": 1.4222,
+      "step": 739
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "grad_norm": 2.647939920425415,
+      "learning_rate": 3.8837638376383765e-06,
+      "loss": 1.5285,
+      "step": 740
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "eval_loss": 1.6907480955123901,
+      "eval_runtime": 45.7678,
+      "eval_samples_per_second": 21.849,
+      "eval_steps_per_second": 0.918,
+      "step": 740
+    },
+    {
+      "epoch": 0.9222153080273802,
+      "grad_norm": 2.7664523124694824,
+      "learning_rate": 3.881457564575646e-06,
+      "loss": 1.5349,
+      "step": 741
+    },
+    {
+      "epoch": 0.9234598630989421,
+      "grad_norm": 2.5770998001098633,
+      "learning_rate": 3.879151291512915e-06,
+      "loss": 1.5071,
+      "step": 742
+    },
+    {
+      "epoch": 0.9247044181705041,
+      "grad_norm": 2.502567768096924,
+      "learning_rate": 3.876845018450185e-06,
+      "loss": 1.492,
+      "step": 743
+    },
+    {
+      "epoch": 0.9259489732420659,
+      "grad_norm": 2.6531119346618652,
+      "learning_rate": 3.874538745387454e-06,
+      "loss": 1.5045,
+      "step": 744
+    },
+    {
+      "epoch": 0.9271935283136279,
+      "grad_norm": 2.7343876361846924,
+      "learning_rate": 3.872232472324724e-06,
+      "loss": 1.5194,
+      "step": 745
+    },
+    {
+      "epoch": 0.9284380833851898,
+      "grad_norm": 2.842313528060913,
+      "learning_rate": 3.869926199261993e-06,
+      "loss": 1.5588,
+      "step": 746
+    },
+    {
+      "epoch": 0.9296826384567517,
+      "grad_norm": 2.718245267868042,
+      "learning_rate": 3.8676199261992625e-06,
+      "loss": 1.5252,
+      "step": 747
+    },
+    {
+      "epoch": 0.9309271935283137,
+      "grad_norm": 2.695392370223999,
+      "learning_rate": 3.865313653136532e-06,
+      "loss": 1.5596,
+      "step": 748
+    },
+    {
+      "epoch": 0.9321717485998755,
+      "grad_norm": 2.7859959602355957,
+      "learning_rate": 3.863007380073801e-06,
+      "loss": 1.5151,
+      "step": 749
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "grad_norm": 2.5199174880981445,
+      "learning_rate": 3.860701107011071e-06,
+      "loss": 1.4948,
+      "step": 750
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "eval_loss": 1.6868674755096436,
+      "eval_runtime": 46.4115,
+      "eval_samples_per_second": 21.546,
+      "eval_steps_per_second": 0.905,
+      "step": 750
+    },
+    {
+      "epoch": 0.9346608587429994,
+      "grad_norm": 2.8194046020507812,
+      "learning_rate": 3.858394833948339e-06,
+      "loss": 1.5144,
+      "step": 751
+    },
+    {
+      "epoch": 0.9359054138145613,
+      "grad_norm": 2.810380697250366,
+      "learning_rate": 3.856088560885609e-06,
+      "loss": 1.4998,
+      "step": 752
+    },
+    {
+      "epoch": 0.9371499688861232,
+      "grad_norm": 2.6906750202178955,
+      "learning_rate": 3.853782287822879e-06,
+      "loss": 1.4748,
+      "step": 753
+    },
+    {
+      "epoch": 0.9383945239576851,
+      "grad_norm": 2.9030277729034424,
+      "learning_rate": 3.851476014760148e-06,
+      "loss": 1.615,
+      "step": 754
+    },
+    {
+      "epoch": 0.939639079029247,
+      "grad_norm": 2.8690176010131836,
+      "learning_rate": 3.849169741697417e-06,
+      "loss": 1.5083,
+      "step": 755
+    },
+    {
+      "epoch": 0.940883634100809,
+      "grad_norm": 2.669646978378296,
+      "learning_rate": 3.8468634686346865e-06,
+      "loss": 1.5572,
+      "step": 756
+    },
+    {
+      "epoch": 0.9421281891723708,
+      "grad_norm": 2.6517696380615234,
+      "learning_rate": 3.844557195571956e-06,
+      "loss": 1.5142,
+      "step": 757
+    },
+    {
+      "epoch": 0.9433727442439328,
+      "grad_norm": 2.5524444580078125,
+      "learning_rate": 3.842250922509225e-06,
+      "loss": 1.4924,
+      "step": 758
+    },
+    {
+      "epoch": 0.9446172993154948,
+      "grad_norm": 2.6532633304595947,
+      "learning_rate": 3.839944649446495e-06,
+      "loss": 1.484,
+      "step": 759
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "grad_norm": 2.7779057025909424,
+      "learning_rate": 3.837638376383764e-06,
+      "loss": 1.5106,
+      "step": 760
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "eval_loss": 1.6845752000808716,
+      "eval_runtime": 50.3415,
+      "eval_samples_per_second": 19.864,
+      "eval_steps_per_second": 0.834,
+      "step": 760
+    },
+    {
+      "epoch": 0.9471064094586186,
+      "grad_norm": 2.6192541122436523,
+      "learning_rate": 3.8353321033210336e-06,
+      "loss": 1.5454,
+      "step": 761
+    },
+    {
+      "epoch": 0.9483509645301804,
+      "grad_norm": 2.561861991882324,
+      "learning_rate": 3.833025830258303e-06,
+      "loss": 1.5141,
+      "step": 762
+    },
+    {
+      "epoch": 0.9495955196017424,
+      "grad_norm": 2.661829948425293,
+      "learning_rate": 3.830719557195572e-06,
+      "loss": 1.4997,
+      "step": 763
+    },
+    {
+      "epoch": 0.9508400746733043,
+      "grad_norm": 2.725275754928589,
+      "learning_rate": 3.828413284132842e-06,
+      "loss": 1.5224,
+      "step": 764
+    },
+    {
+      "epoch": 0.9520846297448662,
+      "grad_norm": 2.5664193630218506,
+      "learning_rate": 3.826107011070111e-06,
+      "loss": 1.5073,
+      "step": 765
+    },
+    {
+      "epoch": 0.9533291848164281,
+      "grad_norm": 2.595189332962036,
+      "learning_rate": 3.823800738007381e-06,
+      "loss": 1.5379,
+      "step": 766
+    },
+    {
+      "epoch": 0.95457373988799,
+      "grad_norm": 2.738060235977173,
+      "learning_rate": 3.821494464944649e-06,
+      "loss": 1.4814,
+      "step": 767
+    },
+    {
+      "epoch": 0.955818294959552,
+      "grad_norm": 2.601071357727051,
+      "learning_rate": 3.819188191881919e-06,
+      "loss": 1.4812,
+      "step": 768
+    },
+    {
+      "epoch": 0.9570628500311139,
+      "grad_norm": 2.657212257385254,
+      "learning_rate": 3.816881918819189e-06,
+      "loss": 1.4748,
+      "step": 769
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "grad_norm": 2.7673351764678955,
+      "learning_rate": 3.814575645756458e-06,
+      "loss": 1.5535,
+      "step": 770
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "eval_loss": 1.677243947982788,
+      "eval_runtime": 51.5537,
+      "eval_samples_per_second": 19.397,
+      "eval_steps_per_second": 0.815,
+      "step": 770
+    },
+    {
+      "epoch": 0.9595519601742377,
+      "grad_norm": 2.760890007019043,
+      "learning_rate": 3.812269372693727e-06,
+      "loss": 1.5654,
+      "step": 771
+    },
+    {
+      "epoch": 0.9607965152457997,
+      "grad_norm": 2.6934309005737305,
+      "learning_rate": 3.809963099630997e-06,
+      "loss": 1.5149,
+      "step": 772
+    },
+    {
+      "epoch": 0.9620410703173615,
+      "grad_norm": 2.729950428009033,
+      "learning_rate": 3.8076568265682662e-06,
+      "loss": 1.5579,
+      "step": 773
+    },
+    {
+      "epoch": 0.9632856253889235,
+      "grad_norm": 2.732926607131958,
+      "learning_rate": 3.8053505535055352e-06,
+      "loss": 1.5179,
+      "step": 774
+    },
+    {
+      "epoch": 0.9645301804604853,
+      "grad_norm": 2.745391368865967,
+      "learning_rate": 3.8030442804428046e-06,
+      "loss": 1.5124,
+      "step": 775
+    },
+    {
+      "epoch": 0.9657747355320473,
+      "grad_norm": 2.8050146102905273,
+      "learning_rate": 3.800738007380074e-06,
+      "loss": 1.519,
+      "step": 776
+    },
+    {
+      "epoch": 0.9670192906036092,
+      "grad_norm": 2.697171449661255,
+      "learning_rate": 3.798431734317343e-06,
+      "loss": 1.5219,
+      "step": 777
+    },
+    {
+      "epoch": 0.9682638456751711,
+      "grad_norm": 2.6375980377197266,
+      "learning_rate": 3.796125461254613e-06,
+      "loss": 1.5345,
+      "step": 778
+    },
+    {
+      "epoch": 0.9695084007467331,
+      "grad_norm": 2.586636781692505,
+      "learning_rate": 3.7938191881918823e-06,
+      "loss": 1.5182,
+      "step": 779
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "grad_norm": 2.626453399658203,
+      "learning_rate": 3.7915129151291518e-06,
+      "loss": 1.4817,
+      "step": 780
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "eval_loss": 1.6734713315963745,
+      "eval_runtime": 48.1436,
+      "eval_samples_per_second": 20.771,
+      "eval_steps_per_second": 0.872,
+      "step": 780
+    },
+    {
+      "epoch": 0.9719975108898569,
+      "grad_norm": 2.579970121383667,
+      "learning_rate": 3.7892066420664208e-06,
+      "loss": 1.4981,
+      "step": 781
+    },
+    {
+      "epoch": 0.9732420659614188,
+      "grad_norm": 2.6834843158721924,
+      "learning_rate": 3.7869003690036906e-06,
+      "loss": 1.5044,
+      "step": 782
+    },
+    {
+      "epoch": 0.9744866210329807,
+      "grad_norm": 2.586982488632202,
+      "learning_rate": 3.78459409594096e-06,
+      "loss": 1.5079,
+      "step": 783
+    },
+    {
+      "epoch": 0.9757311761045426,
+      "grad_norm": 2.910027503967285,
+      "learning_rate": 3.782287822878229e-06,
+      "loss": 1.5695,
+      "step": 784
+    },
+    {
+      "epoch": 0.9769757311761046,
+      "grad_norm": 2.7088494300842285,
+      "learning_rate": 3.7799815498154984e-06,
+      "loss": 1.4986,
+      "step": 785
+    },
+    {
+      "epoch": 0.9782202862476664,
+      "grad_norm": 2.581325054168701,
+      "learning_rate": 3.777675276752768e-06,
+      "loss": 1.5137,
+      "step": 786
+    },
+    {
+      "epoch": 0.9794648413192284,
+      "grad_norm": 2.700709819793701,
+      "learning_rate": 3.775369003690037e-06,
+      "loss": 1.4718,
+      "step": 787
+    },
+    {
+      "epoch": 0.9807093963907902,
+      "grad_norm": 2.6712708473205566,
+      "learning_rate": 3.7730627306273067e-06,
+      "loss": 1.5208,
+      "step": 788
+    },
+    {
+      "epoch": 0.9819539514623522,
+      "grad_norm": 2.6799817085266113,
+      "learning_rate": 3.770756457564576e-06,
+      "loss": 1.5374,
+      "step": 789
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "grad_norm": 2.618988037109375,
+      "learning_rate": 3.768450184501845e-06,
+      "loss": 1.4997,
+      "step": 790
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "eval_loss": 1.6860820055007935,
+      "eval_runtime": 51.7654,
+      "eval_samples_per_second": 19.318,
+      "eval_steps_per_second": 0.811,
+      "step": 790
+    },
+    {
+      "epoch": 0.984443061605476,
+      "grad_norm": 2.6899526119232178,
+      "learning_rate": 3.7661439114391146e-06,
+      "loss": 1.5216,
+      "step": 791
+    },
+    {
+      "epoch": 0.985687616677038,
+      "grad_norm": 2.8003487586975098,
+      "learning_rate": 3.763837638376384e-06,
+      "loss": 1.557,
+      "step": 792
+    },
+    {
+      "epoch": 0.9869321717485999,
+      "grad_norm": 2.744536876678467,
+      "learning_rate": 3.761531365313654e-06,
+      "loss": 1.5355,
+      "step": 793
+    },
+    {
+      "epoch": 0.9881767268201618,
+      "grad_norm": 2.587250232696533,
+      "learning_rate": 3.759225092250923e-06,
+      "loss": 1.5259,
+      "step": 794
+    },
+    {
+      "epoch": 0.9894212818917237,
+      "grad_norm": 2.616291046142578,
+      "learning_rate": 3.7569188191881922e-06,
+      "loss": 1.5176,
+      "step": 795
+    },
+    {
+      "epoch": 0.9906658369632856,
+      "grad_norm": 2.6410577297210693,
+      "learning_rate": 3.7546125461254617e-06,
+      "loss": 1.511,
+      "step": 796
+    },
+    {
+      "epoch": 0.9919103920348475,
+      "grad_norm": 2.577373504638672,
+      "learning_rate": 3.7523062730627307e-06,
+      "loss": 1.4715,
+      "step": 797
+    },
+    {
+      "epoch": 0.9931549471064095,
+      "grad_norm": 2.67305326461792,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 1.4828,
+      "step": 798
+    },
+    {
+      "epoch": 0.9943995021779714,
+      "grad_norm": 3.1631500720977783,
+      "learning_rate": 3.74769372693727e-06,
+      "loss": 1.5602,
+      "step": 799
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "grad_norm": 2.9222350120544434,
+      "learning_rate": 3.745387453874539e-06,
+      "loss": 1.4814,
+      "step": 800
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "eval_loss": 1.6810544729232788,
+      "eval_runtime": 51.1262,
+      "eval_samples_per_second": 19.559,
+      "eval_steps_per_second": 0.821,
+      "step": 800
+    },
+    {
+      "epoch": 0.9968886123210952,
+      "grad_norm": 2.884312629699707,
+      "learning_rate": 3.7430811808118084e-06,
+      "loss": 1.4553,
+      "step": 801
+    },
+    {
+      "epoch": 0.9981331673926571,
+      "grad_norm": 2.712655782699585,
+      "learning_rate": 3.7407749077490778e-06,
+      "loss": 1.5275,
+      "step": 802
+    },
+    {
+      "epoch": 0.9993777224642191,
+      "grad_norm": 2.8700571060180664,
+      "learning_rate": 3.7384686346863468e-06,
+      "loss": 1.4923,
+      "step": 803
+    },
+    {
+      "epoch": 1.000622277535781,
+      "grad_norm": 2.7779245376586914,
+      "learning_rate": 3.7361623616236166e-06,
+      "loss": 1.5224,
+      "step": 804
+    },
+    {
+      "epoch": 1.0018668326073428,
+      "grad_norm": 2.7679810523986816,
+      "learning_rate": 3.733856088560886e-06,
+      "loss": 1.4977,
+      "step": 805
+    },
+    {
+      "epoch": 1.0031113876789048,
+      "grad_norm": 2.797023296356201,
+      "learning_rate": 3.7315498154981555e-06,
+      "loss": 1.4396,
+      "step": 806
+    },
+    {
+      "epoch": 1.0043559427504667,
+      "grad_norm": 3.007962465286255,
+      "learning_rate": 3.7292435424354245e-06,
+      "loss": 1.5015,
+      "step": 807
+    },
+    {
+      "epoch": 1.0056004978220285,
+      "grad_norm": 3.127639055252075,
+      "learning_rate": 3.726937269372694e-06,
+      "loss": 1.5176,
+      "step": 808
+    },
+    {
+      "epoch": 1.0068450528935906,
+      "grad_norm": 2.9001357555389404,
+      "learning_rate": 3.7246309963099637e-06,
+      "loss": 1.4735,
+      "step": 809
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "grad_norm": 2.8245413303375244,
+      "learning_rate": 3.7223247232472327e-06,
+      "loss": 1.486,
+      "step": 810
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "eval_loss": 1.6835130453109741,
+      "eval_runtime": 44.5682,
+      "eval_samples_per_second": 22.438,
+      "eval_steps_per_second": 0.942,
+      "step": 810
+    },
+    {
+      "epoch": 1.0093341630367143,
+      "grad_norm": 2.780172824859619,
+      "learning_rate": 3.720018450184502e-06,
+      "loss": 1.4437,
+      "step": 811
+    },
+    {
+      "epoch": 1.0105787181082764,
+      "grad_norm": 2.6013996601104736,
+      "learning_rate": 3.7177121771217716e-06,
+      "loss": 1.4529,
+      "step": 812
+    },
+    {
+      "epoch": 1.0118232731798382,
+      "grad_norm": 2.831015110015869,
+      "learning_rate": 3.7154059040590406e-06,
+      "loss": 1.4825,
+      "step": 813
+    },
+    {
+      "epoch": 1.0130678282514,
+      "grad_norm": 2.8341829776763916,
+      "learning_rate": 3.71309963099631e-06,
+      "loss": 1.4736,
+      "step": 814
+    },
+    {
+      "epoch": 1.0143123833229621,
+      "grad_norm": 2.769540309906006,
+      "learning_rate": 3.71079335793358e-06,
+      "loss": 1.4724,
+      "step": 815
+    },
+    {
+      "epoch": 1.015556938394524,
+      "grad_norm": 3.0083069801330566,
+      "learning_rate": 3.708487084870849e-06,
+      "loss": 1.4985,
+      "step": 816
+    },
+    {
+      "epoch": 1.0168014934660858,
+      "grad_norm": 3.052168607711792,
+      "learning_rate": 3.7061808118081183e-06,
+      "loss": 1.5193,
+      "step": 817
+    },
+    {
+      "epoch": 1.0180460485376477,
+      "grad_norm": 2.924877405166626,
+      "learning_rate": 3.7038745387453877e-06,
+      "loss": 1.5219,
+      "step": 818
+    },
+    {
+      "epoch": 1.0192906036092098,
+      "grad_norm": 3.150123357772827,
+      "learning_rate": 3.7015682656826576e-06,
+      "loss": 1.5026,
+      "step": 819
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "grad_norm": 2.881655216217041,
+      "learning_rate": 3.6992619926199266e-06,
+      "loss": 1.5011,
+      "step": 820
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "eval_loss": 1.6745303869247437,
+      "eval_runtime": 50.6791,
+      "eval_samples_per_second": 19.732,
+      "eval_steps_per_second": 0.829,
+      "step": 820
+    },
+    {
+      "epoch": 1.0217797137523335,
+      "grad_norm": 2.824249505996704,
+      "learning_rate": 3.696955719557196e-06,
+      "loss": 1.4642,
+      "step": 821
+    },
+    {
+      "epoch": 1.0230242688238955,
+      "grad_norm": 2.859144926071167,
+      "learning_rate": 3.6946494464944654e-06,
+      "loss": 1.4681,
+      "step": 822
+    },
+    {
+      "epoch": 1.0242688238954574,
+      "grad_norm": 2.90312123298645,
+      "learning_rate": 3.6923431734317344e-06,
+      "loss": 1.4926,
+      "step": 823
+    },
+    {
+      "epoch": 1.0255133789670192,
+      "grad_norm": 2.8340659141540527,
+      "learning_rate": 3.690036900369004e-06,
+      "loss": 1.4834,
+      "step": 824
+    },
+    {
+      "epoch": 1.0267579340385813,
+      "grad_norm": 2.9743151664733887,
+      "learning_rate": 3.6877306273062737e-06,
+      "loss": 1.4574,
+      "step": 825
+    },
+    {
+      "epoch": 1.0280024891101431,
+      "grad_norm": 3.032179594039917,
+      "learning_rate": 3.6854243542435427e-06,
+      "loss": 1.5571,
+      "step": 826
+    },
+    {
+      "epoch": 1.029247044181705,
+      "grad_norm": 2.816826343536377,
+      "learning_rate": 3.683118081180812e-06,
+      "loss": 1.455,
+      "step": 827
+    },
+    {
+      "epoch": 1.030491599253267,
+      "grad_norm": 3.046696186065674,
+      "learning_rate": 3.6808118081180815e-06,
+      "loss": 1.4871,
+      "step": 828
+    },
+    {
+      "epoch": 1.031736154324829,
+      "grad_norm": 2.8517391681671143,
+      "learning_rate": 3.678505535055351e-06,
+      "loss": 1.5292,
+      "step": 829
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "grad_norm": 2.752811908721924,
+      "learning_rate": 3.67619926199262e-06,
+      "loss": 1.5091,
+      "step": 830
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "eval_loss": 1.6692287921905518,
+      "eval_runtime": 50.8347,
+      "eval_samples_per_second": 19.672,
+      "eval_steps_per_second": 0.826,
+      "step": 830
+    },
+    {
+      "epoch": 1.0342252644679526,
+      "grad_norm": 2.9220356941223145,
+      "learning_rate": 3.6738929889298898e-06,
+      "loss": 1.4247,
+      "step": 831
+    },
+    {
+      "epoch": 1.0354698195395147,
+      "grad_norm": 2.8903002738952637,
+      "learning_rate": 3.671586715867159e-06,
+      "loss": 1.5595,
+      "step": 832
+    },
+    {
+      "epoch": 1.0367143746110765,
+      "grad_norm": 2.6495542526245117,
+      "learning_rate": 3.669280442804428e-06,
+      "loss": 1.4076,
+      "step": 833
+    },
+    {
+      "epoch": 1.0379589296826384,
+      "grad_norm": 2.880809783935547,
+      "learning_rate": 3.6669741697416976e-06,
+      "loss": 1.5285,
+      "step": 834
+    },
+    {
+      "epoch": 1.0392034847542004,
+      "grad_norm": 2.919142007827759,
+      "learning_rate": 3.6646678966789675e-06,
+      "loss": 1.5165,
+      "step": 835
+    },
+    {
+      "epoch": 1.0404480398257623,
+      "grad_norm": 3.1180522441864014,
+      "learning_rate": 3.6623616236162365e-06,
+      "loss": 1.5224,
+      "step": 836
+    },
+    {
+      "epoch": 1.0416925948973241,
+      "grad_norm": 2.767709255218506,
+      "learning_rate": 3.660055350553506e-06,
+      "loss": 1.4859,
+      "step": 837
+    },
+    {
+      "epoch": 1.0429371499688862,
+      "grad_norm": 2.708828926086426,
+      "learning_rate": 3.6577490774907753e-06,
+      "loss": 1.4803,
+      "step": 838
+    },
+    {
+      "epoch": 1.044181705040448,
+      "grad_norm": 2.688359260559082,
+      "learning_rate": 3.6554428044280443e-06,
+      "loss": 1.4571,
+      "step": 839
+    },
+    {
+      "epoch": 1.04542626011201,
+      "grad_norm": 2.691812753677368,
+      "learning_rate": 3.6531365313653137e-06,
+      "loss": 1.5043,
+      "step": 840
+    },
+    {
+      "epoch": 1.04542626011201,
+      "eval_loss": 1.6769559383392334,
+      "eval_runtime": 49.6233,
+      "eval_samples_per_second": 20.152,
+      "eval_steps_per_second": 0.846,
+      "step": 840
+    },
+    {
+      "epoch": 1.046670815183572,
+      "grad_norm": 2.686372995376587,
+      "learning_rate": 3.6508302583025836e-06,
+      "loss": 1.5221,
+      "step": 841
+    },
+    {
+      "epoch": 1.0479153702551338,
+      "grad_norm": 2.742859125137329,
+      "learning_rate": 3.648523985239853e-06,
+      "loss": 1.4685,
+      "step": 842
+    },
+    {
+      "epoch": 1.0491599253266957,
+      "grad_norm": 2.874018907546997,
+      "learning_rate": 3.646217712177122e-06,
+      "loss": 1.4813,
+      "step": 843
+    },
+    {
+      "epoch": 1.0504044803982575,
+      "grad_norm": 2.8159990310668945,
+      "learning_rate": 3.6439114391143914e-06,
+      "loss": 1.4421,
+      "step": 844
+    },
+    {
+      "epoch": 1.0516490354698196,
+      "grad_norm": 2.714787006378174,
+      "learning_rate": 3.641605166051661e-06,
+      "loss": 1.4879,
+      "step": 845
+    },
+    {
+      "epoch": 1.0528935905413814,
+      "grad_norm": 2.885148525238037,
+      "learning_rate": 3.63929889298893e-06,
+      "loss": 1.4725,
+      "step": 846
+    },
+    {
+      "epoch": 1.0541381456129433,
+      "grad_norm": 2.7272610664367676,
+      "learning_rate": 3.6369926199261997e-06,
+      "loss": 1.4276,
+      "step": 847
+    },
+    {
+      "epoch": 1.0553827006845053,
+      "grad_norm": 2.9114978313446045,
+      "learning_rate": 3.634686346863469e-06,
+      "loss": 1.4194,
+      "step": 848
+    },
+    {
+      "epoch": 1.0566272557560672,
+      "grad_norm": 3.019313335418701,
+      "learning_rate": 3.632380073800738e-06,
+      "loss": 1.5266,
+      "step": 849
+    },
+    {
+      "epoch": 1.057871810827629,
+      "grad_norm": 2.714751958847046,
+      "learning_rate": 3.6300738007380075e-06,
+      "loss": 1.4593,
+      "step": 850
+    },
+    {
+      "epoch": 1.057871810827629,
+      "eval_loss": 1.6760298013687134,
+      "eval_runtime": 52.1681,
+      "eval_samples_per_second": 19.169,
+      "eval_steps_per_second": 0.805,
+      "step": 850
+    },
+    {
+      "epoch": 1.0591163658991911,
+      "grad_norm": 3.0507936477661133,
+      "learning_rate": 3.627767527675277e-06,
+      "loss": 1.5064,
+      "step": 851
+    },
+    {
+      "epoch": 1.060360920970753,
+      "grad_norm": 2.8116295337677,
+      "learning_rate": 3.625461254612546e-06,
+      "loss": 1.5218,
+      "step": 852
+    },
+    {
+      "epoch": 1.0616054760423148,
+      "grad_norm": 2.8801465034484863,
+      "learning_rate": 3.623154981549816e-06,
+      "loss": 1.4641,
+      "step": 853
+    },
+    {
+      "epoch": 1.0628500311138769,
+      "grad_norm": 3.0581862926483154,
+      "learning_rate": 3.6208487084870852e-06,
+      "loss": 1.483,
+      "step": 854
+    },
+    {
+      "epoch": 1.0640945861854387,
+      "grad_norm": 2.898780584335327,
+      "learning_rate": 3.6185424354243547e-06,
+      "loss": 1.4623,
+      "step": 855
+    },
+    {
+      "epoch": 1.0653391412570006,
+      "grad_norm": 2.971820116043091,
+      "learning_rate": 3.6162361623616237e-06,
+      "loss": 1.4455,
+      "step": 856
+    },
+    {
+      "epoch": 1.0665836963285624,
+      "grad_norm": 3.0021913051605225,
+      "learning_rate": 3.6139298892988935e-06,
+      "loss": 1.4878,
+      "step": 857
+    },
+    {
+      "epoch": 1.0678282514001245,
+      "grad_norm": 3.0697457790374756,
+      "learning_rate": 3.611623616236163e-06,
+      "loss": 1.5008,
+      "step": 858
+    },
+    {
+      "epoch": 1.0690728064716863,
+      "grad_norm": 3.0787923336029053,
+      "learning_rate": 3.609317343173432e-06,
+      "loss": 1.4612,
+      "step": 859
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "grad_norm": 2.789339780807495,
+      "learning_rate": 3.6070110701107014e-06,
+      "loss": 1.4287,
+      "step": 860
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "eval_loss": 1.6747506856918335,
+      "eval_runtime": 51.4912,
+      "eval_samples_per_second": 19.421,
+      "eval_steps_per_second": 0.816,
+      "step": 860
+    },
+    {
+      "epoch": 1.0715619166148103,
+      "grad_norm": 2.9622788429260254,
+      "learning_rate": 3.6047047970479708e-06,
+      "loss": 1.4905,
+      "step": 861
+    },
+    {
+      "epoch": 1.072806471686372,
+      "grad_norm": 3.0675268173217773,
+      "learning_rate": 3.6023985239852398e-06,
+      "loss": 1.461,
+      "step": 862
+    },
+    {
+      "epoch": 1.074051026757934,
+      "grad_norm": 2.820996046066284,
+      "learning_rate": 3.6000922509225096e-06,
+      "loss": 1.4948,
+      "step": 863
+    },
+    {
+      "epoch": 1.075295581829496,
+      "grad_norm": 2.833761215209961,
+      "learning_rate": 3.597785977859779e-06,
+      "loss": 1.4683,
+      "step": 864
+    },
+    {
+      "epoch": 1.0765401369010579,
+      "grad_norm": 2.917313814163208,
+      "learning_rate": 3.595479704797048e-06,
+      "loss": 1.5191,
+      "step": 865
+    },
+    {
+      "epoch": 1.0777846919726197,
+      "grad_norm": 2.876166820526123,
+      "learning_rate": 3.5931734317343175e-06,
+      "loss": 1.4854,
+      "step": 866
+    },
+    {
+      "epoch": 1.0790292470441818,
+      "grad_norm": 3.139049768447876,
+      "learning_rate": 3.590867158671587e-06,
+      "loss": 1.5134,
+      "step": 867
+    },
+    {
+      "epoch": 1.0802738021157436,
+      "grad_norm": 2.9956753253936768,
+      "learning_rate": 3.5885608856088567e-06,
+      "loss": 1.5003,
+      "step": 868
+    },
+    {
+      "epoch": 1.0815183571873055,
+      "grad_norm": 2.957181930541992,
+      "learning_rate": 3.5862546125461257e-06,
+      "loss": 1.4588,
+      "step": 869
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "grad_norm": 2.8576438426971436,
+      "learning_rate": 3.583948339483395e-06,
+      "loss": 1.48,
+      "step": 870
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "eval_loss": 1.6724848747253418,
+      "eval_runtime": 50.9669,
+      "eval_samples_per_second": 19.621,
+      "eval_steps_per_second": 0.824,
+      "step": 870
+    },
+    {
+      "epoch": 1.0840074673304294,
+      "grad_norm": 2.8183560371398926,
+      "learning_rate": 3.5816420664206646e-06,
+      "loss": 1.4818,
+      "step": 871
+    },
+    {
+      "epoch": 1.0852520224019913,
+      "grad_norm": 2.7531797885894775,
+      "learning_rate": 3.5793357933579336e-06,
+      "loss": 1.4712,
+      "step": 872
+    },
+    {
+      "epoch": 1.086496577473553,
+      "grad_norm": 2.923962354660034,
+      "learning_rate": 3.5770295202952034e-06,
+      "loss": 1.4864,
+      "step": 873
+    },
+    {
+      "epoch": 1.0877411325451152,
+      "grad_norm": 2.851724147796631,
+      "learning_rate": 3.574723247232473e-06,
+      "loss": 1.4771,
+      "step": 874
+    },
+    {
+      "epoch": 1.088985687616677,
+      "grad_norm": 3.024634838104248,
+      "learning_rate": 3.572416974169742e-06,
+      "loss": 1.4643,
+      "step": 875
+    },
+    {
+      "epoch": 1.0902302426882389,
+      "grad_norm": 2.7100844383239746,
+      "learning_rate": 3.5701107011070113e-06,
+      "loss": 1.4773,
+      "step": 876
+    },
+    {
+      "epoch": 1.091474797759801,
+      "grad_norm": 2.730502128601074,
+      "learning_rate": 3.5678044280442807e-06,
+      "loss": 1.5051,
+      "step": 877
+    },
+    {
+      "epoch": 1.0927193528313628,
+      "grad_norm": 2.747082233428955,
+      "learning_rate": 3.5654981549815497e-06,
+      "loss": 1.5207,
+      "step": 878
+    },
+    {
+      "epoch": 1.0939639079029246,
+      "grad_norm": 2.718358039855957,
+      "learning_rate": 3.5631918819188195e-06,
+      "loss": 1.5067,
+      "step": 879
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "grad_norm": 2.7794573307037354,
+      "learning_rate": 3.560885608856089e-06,
+      "loss": 1.4616,
+      "step": 880
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "eval_loss": 1.6676132678985596,
+      "eval_runtime": 51.6157,
+      "eval_samples_per_second": 19.374,
+      "eval_steps_per_second": 0.814,
+      "step": 880
+    },
+    {
+      "epoch": 1.0964530180460486,
+      "grad_norm": 2.727416753768921,
+      "learning_rate": 3.5585793357933584e-06,
+      "loss": 1.4043,
+      "step": 881
+    },
+    {
+      "epoch": 1.0976975731176104,
+      "grad_norm": 2.856900930404663,
+      "learning_rate": 3.5562730627306274e-06,
+      "loss": 1.5198,
+      "step": 882
+    },
+    {
+      "epoch": 1.0989421281891725,
+      "grad_norm": 2.872823715209961,
+      "learning_rate": 3.553966789667897e-06,
+      "loss": 1.4981,
+      "step": 883
+    },
+    {
+      "epoch": 1.1001866832607343,
+      "grad_norm": 2.7724361419677734,
+      "learning_rate": 3.5516605166051667e-06,
+      "loss": 1.4819,
+      "step": 884
+    },
+    {
+      "epoch": 1.1014312383322962,
+      "grad_norm": 2.884434461593628,
+      "learning_rate": 3.5493542435424357e-06,
+      "loss": 1.433,
+      "step": 885
+    },
+    {
+      "epoch": 1.102675793403858,
+      "grad_norm": 2.887873411178589,
+      "learning_rate": 3.547047970479705e-06,
+      "loss": 1.4677,
+      "step": 886
+    },
+    {
+      "epoch": 1.10392034847542,
+      "grad_norm": 2.946845054626465,
+      "learning_rate": 3.5447416974169745e-06,
+      "loss": 1.4983,
+      "step": 887
+    },
+    {
+      "epoch": 1.105164903546982,
+      "grad_norm": 2.752552032470703,
+      "learning_rate": 3.5424354243542435e-06,
+      "loss": 1.4766,
+      "step": 888
+    },
+    {
+      "epoch": 1.1064094586185438,
+      "grad_norm": 2.788634777069092,
+      "learning_rate": 3.5401291512915133e-06,
+      "loss": 1.4747,
+      "step": 889
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "grad_norm": 2.782792568206787,
+      "learning_rate": 3.5378228782287828e-06,
+      "loss": 1.5503,
+      "step": 890
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "eval_loss": 1.6698333024978638,
+      "eval_runtime": 49.1373,
+      "eval_samples_per_second": 20.351,
+      "eval_steps_per_second": 0.855,
+      "step": 890
+    },
+    {
+      "epoch": 1.1088985687616677,
+      "grad_norm": 2.8351166248321533,
+      "learning_rate": 3.535516605166052e-06,
+      "loss": 1.4589,
+      "step": 891
+    },
+    {
+      "epoch": 1.1101431238332296,
+      "grad_norm": 2.798872709274292,
+      "learning_rate": 3.533210332103321e-06,
+      "loss": 1.4496,
+      "step": 892
+    },
+    {
+      "epoch": 1.1113876789047916,
+      "grad_norm": 3.0863239765167236,
+      "learning_rate": 3.5309040590405906e-06,
+      "loss": 1.4698,
+      "step": 893
+    },
+    {
+      "epoch": 1.1126322339763535,
+      "grad_norm": 2.853862762451172,
+      "learning_rate": 3.5285977859778605e-06,
+      "loss": 1.4429,
+      "step": 894
+    },
+    {
+      "epoch": 1.1138767890479153,
+      "grad_norm": 2.8946170806884766,
+      "learning_rate": 3.5262915129151295e-06,
+      "loss": 1.4697,
+      "step": 895
+    },
+    {
+      "epoch": 1.1151213441194774,
+      "grad_norm": 3.043823003768921,
+      "learning_rate": 3.523985239852399e-06,
+      "loss": 1.4666,
+      "step": 896
+    },
+    {
+      "epoch": 1.1163658991910392,
+      "grad_norm": 2.7822980880737305,
+      "learning_rate": 3.5216789667896683e-06,
+      "loss": 1.4523,
+      "step": 897
+    },
+    {
+      "epoch": 1.117610454262601,
+      "grad_norm": 2.9454171657562256,
+      "learning_rate": 3.5193726937269373e-06,
+      "loss": 1.5009,
+      "step": 898
+    },
+    {
+      "epoch": 1.1188550093341632,
+      "grad_norm": 2.8533740043640137,
+      "learning_rate": 3.5170664206642067e-06,
+      "loss": 1.4265,
+      "step": 899
+    },
+    {
+      "epoch": 1.120099564405725,
+      "grad_norm": 2.894937038421631,
+      "learning_rate": 3.5147601476014766e-06,
+      "loss": 1.4895,
+      "step": 900
+    },
+    {
+      "epoch": 1.120099564405725,
+      "eval_loss": 1.6701407432556152,
+      "eval_runtime": 45.477,
+      "eval_samples_per_second": 21.989,
+      "eval_steps_per_second": 0.924,
+      "step": 900
+    },
+    {
+      "epoch": 1.1213441194772868,
+      "grad_norm": 2.933821201324463,
+      "learning_rate": 3.5124538745387456e-06,
+      "loss": 1.468,
+      "step": 901
+    },
+    {
+      "epoch": 1.1225886745488487,
+      "grad_norm": 2.834265947341919,
+      "learning_rate": 3.510147601476015e-06,
+      "loss": 1.4621,
+      "step": 902
+    },
+    {
+      "epoch": 1.1238332296204108,
+      "grad_norm": 2.9064722061157227,
+      "learning_rate": 3.5078413284132844e-06,
+      "loss": 1.504,
+      "step": 903
+    },
+    {
+      "epoch": 1.1250777846919726,
+      "grad_norm": 2.8753886222839355,
+      "learning_rate": 3.505535055350554e-06,
+      "loss": 1.4643,
+      "step": 904
+    },
+    {
+      "epoch": 1.1263223397635345,
+      "grad_norm": 2.894659996032715,
+      "learning_rate": 3.503228782287823e-06,
+      "loss": 1.4909,
+      "step": 905
+    },
+    {
+      "epoch": 1.1275668948350965,
+      "grad_norm": 3.085977792739868,
+      "learning_rate": 3.5009225092250927e-06,
+      "loss": 1.459,
+      "step": 906
+    },
+    {
+      "epoch": 1.1288114499066584,
+      "grad_norm": 2.9958064556121826,
+      "learning_rate": 3.498616236162362e-06,
+      "loss": 1.4152,
+      "step": 907
+    },
+    {
+      "epoch": 1.1300560049782202,
+      "grad_norm": 2.9136762619018555,
+      "learning_rate": 3.496309963099631e-06,
+      "loss": 1.473,
+      "step": 908
+    },
+    {
+      "epoch": 1.1313005600497823,
+      "grad_norm": 2.956437826156616,
+      "learning_rate": 3.4940036900369005e-06,
+      "loss": 1.5167,
+      "step": 909
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "grad_norm": 2.7354865074157715,
+      "learning_rate": 3.4916974169741704e-06,
+      "loss": 1.4413,
+      "step": 910
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "eval_loss": 1.6629912853240967,
+      "eval_runtime": 41.5478,
+      "eval_samples_per_second": 24.069,
+      "eval_steps_per_second": 1.011,
+      "step": 910
+    },
+    {
+      "epoch": 1.133789670192906,
+      "grad_norm": 2.7443008422851562,
+      "learning_rate": 3.4893911439114394e-06,
+      "loss": 1.4475,
+      "step": 911
+    },
+    {
+      "epoch": 1.135034225264468,
+      "grad_norm": 2.7811238765716553,
+      "learning_rate": 3.487084870848709e-06,
+      "loss": 1.4257,
+      "step": 912
+    },
+    {
+      "epoch": 1.13627878033603,
+      "grad_norm": 2.7916479110717773,
+      "learning_rate": 3.4847785977859782e-06,
+      "loss": 1.4048,
+      "step": 913
+    },
+    {
+      "epoch": 1.1375233354075918,
+      "grad_norm": 2.7835633754730225,
+      "learning_rate": 3.4824723247232472e-06,
+      "loss": 1.3996,
+      "step": 914
+    },
+    {
+      "epoch": 1.1387678904791536,
+      "grad_norm": 2.8188705444335938,
+      "learning_rate": 3.4801660516605166e-06,
+      "loss": 1.4541,
+      "step": 915
+    },
+    {
+      "epoch": 1.1400124455507157,
+      "grad_norm": 3.0134100914001465,
+      "learning_rate": 3.4778597785977865e-06,
+      "loss": 1.4353,
+      "step": 916
+    },
+    {
+      "epoch": 1.1412570006222775,
+      "grad_norm": 2.6965606212615967,
+      "learning_rate": 3.475553505535056e-06,
+      "loss": 1.4308,
+      "step": 917
+    },
+    {
+      "epoch": 1.1425015556938394,
+      "grad_norm": 2.8179919719696045,
+      "learning_rate": 3.473247232472325e-06,
+      "loss": 1.5204,
+      "step": 918
+    },
+    {
+      "epoch": 1.1437461107654014,
+      "grad_norm": 2.7213902473449707,
+      "learning_rate": 3.4709409594095943e-06,
+      "loss": 1.4344,
+      "step": 919
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "grad_norm": 2.850193738937378,
+      "learning_rate": 3.4686346863468638e-06,
+      "loss": 1.473,
+      "step": 920
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "eval_loss": 1.663345217704773,
+      "eval_runtime": 43.8397,
+      "eval_samples_per_second": 22.81,
+      "eval_steps_per_second": 0.958,
+      "step": 920
+    },
+    {
+      "epoch": 1.1462352209085251,
+      "grad_norm": 2.780198574066162,
+      "learning_rate": 3.4663284132841328e-06,
+      "loss": 1.4472,
+      "step": 921
+    },
+    {
+      "epoch": 1.1474797759800872,
+      "grad_norm": 2.9373831748962402,
+      "learning_rate": 3.4640221402214026e-06,
+      "loss": 1.426,
+      "step": 922
+    },
+    {
+      "epoch": 1.148724331051649,
+      "grad_norm": 2.6986873149871826,
+      "learning_rate": 3.461715867158672e-06,
+      "loss": 1.3953,
+      "step": 923
+    },
+    {
+      "epoch": 1.149968886123211,
+      "grad_norm": 2.927441120147705,
+      "learning_rate": 3.459409594095941e-06,
+      "loss": 1.4578,
+      "step": 924
+    },
+    {
+      "epoch": 1.151213441194773,
+      "grad_norm": 2.8469648361206055,
+      "learning_rate": 3.4571033210332105e-06,
+      "loss": 1.4313,
+      "step": 925
+    },
+    {
+      "epoch": 1.1524579962663348,
+      "grad_norm": 2.8114013671875,
+      "learning_rate": 3.4547970479704803e-06,
+      "loss": 1.4267,
+      "step": 926
+    },
+    {
+      "epoch": 1.1537025513378967,
+      "grad_norm": 2.977605104446411,
+      "learning_rate": 3.4524907749077493e-06,
+      "loss": 1.4747,
+      "step": 927
+    },
+    {
+      "epoch": 1.1549471064094585,
+      "grad_norm": 2.9327852725982666,
+      "learning_rate": 3.4501845018450187e-06,
+      "loss": 1.4219,
+      "step": 928
+    },
+    {
+      "epoch": 1.1561916614810206,
+      "grad_norm": 2.844038963317871,
+      "learning_rate": 3.447878228782288e-06,
+      "loss": 1.5146,
+      "step": 929
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "grad_norm": 2.9402201175689697,
+      "learning_rate": 3.4455719557195576e-06,
+      "loss": 1.45,
+      "step": 930
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "eval_loss": 1.6620122194290161,
+      "eval_runtime": 46.1748,
+      "eval_samples_per_second": 21.657,
+      "eval_steps_per_second": 0.91,
+      "step": 930
+    },
+    {
+      "epoch": 1.1586807716241443,
+      "grad_norm": 2.842651128768921,
+      "learning_rate": 3.4432656826568266e-06,
+      "loss": 1.4415,
+      "step": 931
+    },
+    {
+      "epoch": 1.1599253266957064,
+      "grad_norm": 2.8909685611724854,
+      "learning_rate": 3.4409594095940964e-06,
+      "loss": 1.4529,
+      "step": 932
+    },
+    {
+      "epoch": 1.1611698817672682,
+      "grad_norm": 2.8025808334350586,
+      "learning_rate": 3.438653136531366e-06,
+      "loss": 1.4552,
+      "step": 933
+    },
+    {
+      "epoch": 1.16241443683883,
+      "grad_norm": 2.6995794773101807,
+      "learning_rate": 3.436346863468635e-06,
+      "loss": 1.4435,
+      "step": 934
+    },
+    {
+      "epoch": 1.1636589919103921,
+      "grad_norm": 2.7307121753692627,
+      "learning_rate": 3.4340405904059043e-06,
+      "loss": 1.4447,
+      "step": 935
+    },
+    {
+      "epoch": 1.164903546981954,
+      "grad_norm": 2.9532501697540283,
+      "learning_rate": 3.4317343173431737e-06,
+      "loss": 1.4747,
+      "step": 936
+    },
+    {
+      "epoch": 1.1661481020535158,
+      "grad_norm": 3.0135788917541504,
+      "learning_rate": 3.4294280442804427e-06,
+      "loss": 1.525,
+      "step": 937
+    },
+    {
+      "epoch": 1.167392657125078,
+      "grad_norm": 2.796355962753296,
+      "learning_rate": 3.4271217712177125e-06,
+      "loss": 1.4539,
+      "step": 938
+    },
+    {
+      "epoch": 1.1686372121966397,
+      "grad_norm": 2.9422309398651123,
+      "learning_rate": 3.424815498154982e-06,
+      "loss": 1.4664,
+      "step": 939
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "grad_norm": 2.7680039405822754,
+      "learning_rate": 3.4225092250922514e-06,
+      "loss": 1.4837,
+      "step": 940
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "eval_loss": 1.660908818244934,
+      "eval_runtime": 45.6998,
+      "eval_samples_per_second": 21.882,
+      "eval_steps_per_second": 0.919,
+      "step": 940
+    },
+    {
+      "epoch": 1.1711263223397634,
+      "grad_norm": 2.8888845443725586,
+      "learning_rate": 3.4202029520295204e-06,
+      "loss": 1.5315,
+      "step": 941
+    },
+    {
+      "epoch": 1.1723708774113255,
+      "grad_norm": 2.7124130725860596,
+      "learning_rate": 3.41789667896679e-06,
+      "loss": 1.4188,
+      "step": 942
+    },
+    {
+      "epoch": 1.1736154324828874,
+      "grad_norm": 2.75203013420105,
+      "learning_rate": 3.4155904059040596e-06,
+      "loss": 1.4533,
+      "step": 943
+    },
+    {
+      "epoch": 1.1748599875544492,
+      "grad_norm": 2.865480422973633,
+      "learning_rate": 3.4132841328413286e-06,
+      "loss": 1.4532,
+      "step": 944
+    },
+    {
+      "epoch": 1.1761045426260113,
+      "grad_norm": 2.8618321418762207,
+      "learning_rate": 3.410977859778598e-06,
+      "loss": 1.4386,
+      "step": 945
+    },
+    {
+      "epoch": 1.1773490976975731,
+      "grad_norm": 2.7656145095825195,
+      "learning_rate": 3.4086715867158675e-06,
+      "loss": 1.3996,
+      "step": 946
+    },
+    {
+      "epoch": 1.178593652769135,
+      "grad_norm": 2.76213002204895,
+      "learning_rate": 3.4063653136531365e-06,
+      "loss": 1.4979,
+      "step": 947
+    },
+    {
+      "epoch": 1.179838207840697,
+      "grad_norm": 2.8581957817077637,
+      "learning_rate": 3.4040590405904063e-06,
+      "loss": 1.4673,
+      "step": 948
+    },
+    {
+      "epoch": 1.1810827629122589,
+      "grad_norm": 2.7825586795806885,
+      "learning_rate": 3.4017527675276758e-06,
+      "loss": 1.4453,
+      "step": 949
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "grad_norm": 2.9226131439208984,
+      "learning_rate": 3.3994464944649448e-06,
+      "loss": 1.4879,
+      "step": 950
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "eval_loss": 1.6621263027191162,
+      "eval_runtime": 45.1222,
+      "eval_samples_per_second": 22.162,
+      "eval_steps_per_second": 0.931,
+      "step": 950
+    },
+    {
+      "epoch": 1.1835718730553828,
+      "grad_norm": 2.9997975826263428,
+      "learning_rate": 3.397140221402214e-06,
+      "loss": 1.505,
+      "step": 951
+    },
+    {
+      "epoch": 1.1848164281269447,
+      "grad_norm": 2.7225453853607178,
+      "learning_rate": 3.3948339483394836e-06,
+      "loss": 1.4695,
+      "step": 952
+    },
+    {
+      "epoch": 1.1860609831985065,
+      "grad_norm": 2.916473150253296,
+      "learning_rate": 3.3925276752767534e-06,
+      "loss": 1.4901,
+      "step": 953
+    },
+    {
+      "epoch": 1.1873055382700684,
+      "grad_norm": 2.8371644020080566,
+      "learning_rate": 3.3902214022140224e-06,
+      "loss": 1.4944,
+      "step": 954
+    },
+    {
+      "epoch": 1.1885500933416304,
+      "grad_norm": 2.813662528991699,
+      "learning_rate": 3.387915129151292e-06,
+      "loss": 1.4603,
+      "step": 955
+    },
+    {
+      "epoch": 1.1897946484131923,
+      "grad_norm": 2.991931676864624,
+      "learning_rate": 3.3856088560885613e-06,
+      "loss": 1.5496,
+      "step": 956
+    },
+    {
+      "epoch": 1.1910392034847541,
+      "grad_norm": 2.6252501010894775,
+      "learning_rate": 3.3833025830258303e-06,
+      "loss": 1.4405,
+      "step": 957
+    },
+    {
+      "epoch": 1.1922837585563162,
+      "grad_norm": 2.747725009918213,
+      "learning_rate": 3.3809963099630997e-06,
+      "loss": 1.4689,
+      "step": 958
+    },
+    {
+      "epoch": 1.193528313627878,
+      "grad_norm": 2.887763261795044,
+      "learning_rate": 3.3786900369003696e-06,
+      "loss": 1.4501,
+      "step": 959
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "grad_norm": 2.9954354763031006,
+      "learning_rate": 3.3763837638376386e-06,
+      "loss": 1.4978,
+      "step": 960
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "eval_loss": 1.664442539215088,
+      "eval_runtime": 49.1266,
+      "eval_samples_per_second": 20.356,
+      "eval_steps_per_second": 0.855,
+      "step": 960
+    },
+    {
+      "epoch": 1.196017423771002,
+      "grad_norm": 3.0174152851104736,
+      "learning_rate": 3.374077490774908e-06,
+      "loss": 1.4353,
+      "step": 961
+    },
+    {
+      "epoch": 1.1972619788425638,
+      "grad_norm": 2.9082882404327393,
+      "learning_rate": 3.3717712177121774e-06,
+      "loss": 1.4878,
+      "step": 962
+    },
+    {
+      "epoch": 1.1985065339141256,
+      "grad_norm": 2.897993326187134,
+      "learning_rate": 3.3694649446494464e-06,
+      "loss": 1.4975,
+      "step": 963
+    },
+    {
+      "epoch": 1.1997510889856877,
+      "grad_norm": 2.739470958709717,
+      "learning_rate": 3.3671586715867163e-06,
+      "loss": 1.4202,
+      "step": 964
+    },
+    {
+      "epoch": 1.2009956440572496,
+      "grad_norm": 2.817786693572998,
+      "learning_rate": 3.3648523985239857e-06,
+      "loss": 1.5195,
+      "step": 965
+    },
+    {
+      "epoch": 1.2022401991288114,
+      "grad_norm": 2.8476386070251465,
+      "learning_rate": 3.362546125461255e-06,
+      "loss": 1.444,
+      "step": 966
+    },
+    {
+      "epoch": 1.2034847542003733,
+      "grad_norm": 2.804047107696533,
+      "learning_rate": 3.360239852398524e-06,
+      "loss": 1.4442,
+      "step": 967
+    },
+    {
+      "epoch": 1.2047293092719353,
+      "grad_norm": 2.9285430908203125,
+      "learning_rate": 3.3579335793357935e-06,
+      "loss": 1.4877,
+      "step": 968
+    },
+    {
+      "epoch": 1.2059738643434972,
+      "grad_norm": 2.8814878463745117,
+      "learning_rate": 3.3556273062730634e-06,
+      "loss": 1.5078,
+      "step": 969
+    },
+    {
+      "epoch": 1.207218419415059,
+      "grad_norm": 2.970728635787964,
+      "learning_rate": 3.3533210332103324e-06,
+      "loss": 1.4166,
+      "step": 970
+    },
+    {
+      "epoch": 1.207218419415059,
+      "eval_loss": 1.6635315418243408,
+      "eval_runtime": 47.2321,
+      "eval_samples_per_second": 21.172,
+      "eval_steps_per_second": 0.889,
+      "step": 970
+    },
+    {
+      "epoch": 1.208462974486621,
+      "grad_norm": 2.9836292266845703,
+      "learning_rate": 3.3510147601476018e-06,
+      "loss": 1.4777,
+      "step": 971
+    },
+    {
+      "epoch": 1.209707529558183,
+      "grad_norm": 2.8495216369628906,
+      "learning_rate": 3.348708487084871e-06,
+      "loss": 1.4829,
+      "step": 972
+    },
+    {
+      "epoch": 1.2109520846297448,
+      "grad_norm": 2.7709312438964844,
+      "learning_rate": 3.34640221402214e-06,
+      "loss": 1.4242,
+      "step": 973
+    },
+    {
+      "epoch": 1.2121966397013069,
+      "grad_norm": 2.999462366104126,
+      "learning_rate": 3.3440959409594096e-06,
+      "loss": 1.4148,
+      "step": 974
+    },
+    {
+      "epoch": 1.2134411947728687,
+      "grad_norm": 2.8719074726104736,
+      "learning_rate": 3.3417896678966795e-06,
+      "loss": 1.4196,
+      "step": 975
+    },
+    {
+      "epoch": 1.2146857498444306,
+      "grad_norm": 2.8469338417053223,
+      "learning_rate": 3.3394833948339485e-06,
+      "loss": 1.4941,
+      "step": 976
+    },
+    {
+      "epoch": 1.2159303049159926,
+      "grad_norm": 2.9780938625335693,
+      "learning_rate": 3.337177121771218e-06,
+      "loss": 1.513,
+      "step": 977
+    },
+    {
+      "epoch": 1.2171748599875545,
+      "grad_norm": 2.814760684967041,
+      "learning_rate": 3.3348708487084873e-06,
+      "loss": 1.4204,
+      "step": 978
+    },
+    {
+      "epoch": 1.2184194150591163,
+      "grad_norm": 3.016261339187622,
+      "learning_rate": 3.332564575645757e-06,
+      "loss": 1.4389,
+      "step": 979
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "grad_norm": 2.9385855197906494,
+      "learning_rate": 3.330258302583026e-06,
+      "loss": 1.4885,
+      "step": 980
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "eval_loss": 1.6688873767852783,
+      "eval_runtime": 46.2942,
+      "eval_samples_per_second": 21.601,
+      "eval_steps_per_second": 0.907,
+      "step": 980
+    },
+    {
+      "epoch": 1.2209085252022402,
+      "grad_norm": 3.1434757709503174,
+      "learning_rate": 3.3279520295202956e-06,
+      "loss": 1.5157,
+      "step": 981
+    },
+    {
+      "epoch": 1.222153080273802,
+      "grad_norm": 3.0104827880859375,
+      "learning_rate": 3.325645756457565e-06,
+      "loss": 1.49,
+      "step": 982
+    },
+    {
+      "epoch": 1.223397635345364,
+      "grad_norm": 3.1913645267486572,
+      "learning_rate": 3.323339483394834e-06,
+      "loss": 1.4937,
+      "step": 983
+    },
+    {
+      "epoch": 1.224642190416926,
+      "grad_norm": 3.19496750831604,
+      "learning_rate": 3.3210332103321034e-06,
+      "loss": 1.5197,
+      "step": 984
+    },
+    {
+      "epoch": 1.2258867454884879,
+      "grad_norm": 3.046232223510742,
+      "learning_rate": 3.3187269372693733e-06,
+      "loss": 1.4497,
+      "step": 985
+    },
+    {
+      "epoch": 1.2271313005600497,
+      "grad_norm": 2.850675344467163,
+      "learning_rate": 3.3164206642066423e-06,
+      "loss": 1.4595,
+      "step": 986
+    },
+    {
+      "epoch": 1.2283758556316118,
+      "grad_norm": 2.7744576930999756,
+      "learning_rate": 3.3141143911439117e-06,
+      "loss": 1.4726,
+      "step": 987
+    },
+    {
+      "epoch": 1.2296204107031736,
+      "grad_norm": 3.017153024673462,
+      "learning_rate": 3.311808118081181e-06,
+      "loss": 1.493,
+      "step": 988
+    },
+    {
+      "epoch": 1.2308649657747355,
+      "grad_norm": 2.8821120262145996,
+      "learning_rate": 3.3095018450184506e-06,
+      "loss": 1.4734,
+      "step": 989
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "grad_norm": 2.8805840015411377,
+      "learning_rate": 3.3071955719557196e-06,
+      "loss": 1.4562,
+      "step": 990
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "eval_loss": 1.669345736503601,
+      "eval_runtime": 47.0893,
+      "eval_samples_per_second": 21.236,
+      "eval_steps_per_second": 0.892,
+      "step": 990
+    },
+    {
+      "epoch": 1.2333540759178594,
+      "grad_norm": 2.7599008083343506,
+      "learning_rate": 3.3048892988929894e-06,
+      "loss": 1.4218,
+      "step": 991
+    },
+    {
+      "epoch": 1.2345986309894212,
+      "grad_norm": 2.9835240840911865,
+      "learning_rate": 3.302583025830259e-06,
+      "loss": 1.4596,
+      "step": 992
+    },
+    {
+      "epoch": 1.235843186060983,
+      "grad_norm": 2.843358039855957,
+      "learning_rate": 3.300276752767528e-06,
+      "loss": 1.4561,
+      "step": 993
+    },
+    {
+      "epoch": 1.2370877411325452,
+      "grad_norm": 2.7386093139648438,
+      "learning_rate": 3.2979704797047972e-06,
+      "loss": 1.4475,
+      "step": 994
+    },
+    {
+      "epoch": 1.238332296204107,
+      "grad_norm": 2.668161392211914,
+      "learning_rate": 3.2956642066420667e-06,
+      "loss": 1.4661,
+      "step": 995
+    },
+    {
+      "epoch": 1.2395768512756689,
+      "grad_norm": 2.947235107421875,
+      "learning_rate": 3.2933579335793357e-06,
+      "loss": 1.4358,
+      "step": 996
+    },
+    {
+      "epoch": 1.240821406347231,
+      "grad_norm": 2.8159878253936768,
+      "learning_rate": 3.2910516605166055e-06,
+      "loss": 1.4432,
+      "step": 997
+    },
+    {
+      "epoch": 1.2420659614187928,
+      "grad_norm": 2.896597385406494,
+      "learning_rate": 3.288745387453875e-06,
+      "loss": 1.5585,
+      "step": 998
+    },
+    {
+      "epoch": 1.2433105164903546,
+      "grad_norm": 2.80241322517395,
+      "learning_rate": 3.286439114391144e-06,
+      "loss": 1.5076,
+      "step": 999
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "grad_norm": 3.008939027786255,
+      "learning_rate": 3.2841328413284134e-06,
+      "loss": 1.4234,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "eval_loss": 1.6621588468551636,
+      "eval_runtime": 50.4926,
+      "eval_samples_per_second": 19.805,
+      "eval_steps_per_second": 0.832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2457996266334785,
+      "grad_norm": 2.9872381687164307,
+      "learning_rate": 3.281826568265683e-06,
+      "loss": 1.453,
+      "step": 1001
+    },
+    {
+      "epoch": 1.2470441817050404,
+      "grad_norm": 2.704676628112793,
+      "learning_rate": 3.2795202952029526e-06,
+      "loss": 1.4711,
+      "step": 1002
+    },
+    {
+      "epoch": 1.2482887367766025,
+      "grad_norm": 2.8100757598876953,
+      "learning_rate": 3.2772140221402216e-06,
+      "loss": 1.3884,
+      "step": 1003
+    },
+    {
+      "epoch": 1.2495332918481643,
+      "grad_norm": 2.902953624725342,
+      "learning_rate": 3.274907749077491e-06,
+      "loss": 1.4986,
+      "step": 1004
+    },
+    {
+      "epoch": 1.2507778469197262,
+      "grad_norm": 2.980163097381592,
+      "learning_rate": 3.2726014760147605e-06,
+      "loss": 1.5067,
+      "step": 1005
+    },
+    {
+      "epoch": 1.252022401991288,
+      "grad_norm": 2.8820345401763916,
+      "learning_rate": 3.2702952029520295e-06,
+      "loss": 1.4995,
+      "step": 1006
+    },
+    {
+      "epoch": 1.25326695706285,
+      "grad_norm": 2.8219635486602783,
+      "learning_rate": 3.2679889298892993e-06,
+      "loss": 1.4881,
+      "step": 1007
+    },
+    {
+      "epoch": 1.254511512134412,
+      "grad_norm": 2.8593199253082275,
+      "learning_rate": 3.2656826568265687e-06,
+      "loss": 1.4469,
+      "step": 1008
+    },
+    {
+      "epoch": 1.255756067205974,
+      "grad_norm": 2.888357639312744,
+      "learning_rate": 3.2633763837638377e-06,
+      "loss": 1.4869,
+      "step": 1009
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "grad_norm": 2.9985013008117676,
+      "learning_rate": 3.261070110701107e-06,
+      "loss": 1.4724,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "eval_loss": 1.6492141485214233,
+      "eval_runtime": 45.2197,
+      "eval_samples_per_second": 22.114,
+      "eval_steps_per_second": 0.929,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2582451773490977,
+      "grad_norm": 2.860780715942383,
+      "learning_rate": 3.2587638376383766e-06,
+      "loss": 1.4618,
+      "step": 1011
+    },
+    {
+      "epoch": 1.2594897324206595,
+      "grad_norm": 2.766479730606079,
+      "learning_rate": 3.2564575645756456e-06,
+      "loss": 1.4642,
+      "step": 1012
+    },
+    {
+      "epoch": 1.2607342874922216,
+      "grad_norm": 2.711759328842163,
+      "learning_rate": 3.2541512915129154e-06,
+      "loss": 1.4283,
+      "step": 1013
+    },
+    {
+      "epoch": 1.2619788425637835,
+      "grad_norm": 2.8365769386291504,
+      "learning_rate": 3.251845018450185e-06,
+      "loss": 1.4248,
+      "step": 1014
+    },
+    {
+      "epoch": 1.2632233976353453,
+      "grad_norm": 2.957620143890381,
+      "learning_rate": 3.2495387453874543e-06,
+      "loss": 1.4662,
+      "step": 1015
+    },
+    {
+      "epoch": 1.2644679527069074,
+      "grad_norm": 2.8066208362579346,
+      "learning_rate": 3.2472324723247233e-06,
+      "loss": 1.4447,
+      "step": 1016
+    },
+    {
+      "epoch": 1.2657125077784692,
+      "grad_norm": 2.902040958404541,
+      "learning_rate": 3.244926199261993e-06,
+      "loss": 1.4743,
+      "step": 1017
+    },
+    {
+      "epoch": 1.266957062850031,
+      "grad_norm": 2.7636687755584717,
+      "learning_rate": 3.2426199261992625e-06,
+      "loss": 1.4465,
+      "step": 1018
+    },
+    {
+      "epoch": 1.268201617921593,
+      "grad_norm": 2.95261549949646,
+      "learning_rate": 3.2403136531365315e-06,
+      "loss": 1.4769,
+      "step": 1019
+    },
+    {
+      "epoch": 1.269446172993155,
+      "grad_norm": 2.911128520965576,
+      "learning_rate": 3.238007380073801e-06,
+      "loss": 1.3843,
+      "step": 1020
+    },
+    {
+      "epoch": 1.269446172993155,
+      "eval_loss": 1.6526964902877808,
+      "eval_runtime": 48.2126,
+      "eval_samples_per_second": 20.741,
+      "eval_steps_per_second": 0.871,
+      "step": 1020
+    },
+    {
+      "epoch": 1.2706907280647168,
+      "grad_norm": 2.769951820373535,
+      "learning_rate": 3.2357011070110704e-06,
+      "loss": 1.4847,
+      "step": 1021
+    },
+    {
+      "epoch": 1.271935283136279,
+      "grad_norm": 3.0451674461364746,
+      "learning_rate": 3.2333948339483394e-06,
+      "loss": 1.4453,
+      "step": 1022
+    },
+    {
+      "epoch": 1.2731798382078408,
+      "grad_norm": 2.9543521404266357,
+      "learning_rate": 3.2310885608856092e-06,
+      "loss": 1.4789,
+      "step": 1023
+    },
+    {
+      "epoch": 1.2744243932794026,
+      "grad_norm": 2.9174394607543945,
+      "learning_rate": 3.2287822878228787e-06,
+      "loss": 1.4807,
+      "step": 1024
+    },
+    {
+      "epoch": 1.2756689483509644,
+      "grad_norm": 2.888258934020996,
+      "learning_rate": 3.2264760147601477e-06,
+      "loss": 1.4367,
+      "step": 1025
+    },
+    {
+      "epoch": 1.2769135034225265,
+      "grad_norm": 2.8346047401428223,
+      "learning_rate": 3.224169741697417e-06,
+      "loss": 1.4313,
+      "step": 1026
+    },
+    {
+      "epoch": 1.2781580584940884,
+      "grad_norm": 2.7531332969665527,
+      "learning_rate": 3.2218634686346865e-06,
+      "loss": 1.4018,
+      "step": 1027
+    },
+    {
+      "epoch": 1.2794026135656502,
+      "grad_norm": 3.136946439743042,
+      "learning_rate": 3.2195571955719564e-06,
+      "loss": 1.4417,
+      "step": 1028
+    },
+    {
+      "epoch": 1.2806471686372123,
+      "grad_norm": 2.9618165493011475,
+      "learning_rate": 3.2172509225092254e-06,
+      "loss": 1.5117,
+      "step": 1029
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "grad_norm": 3.0740747451782227,
+      "learning_rate": 3.2149446494464948e-06,
+      "loss": 1.4641,
+      "step": 1030
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "eval_loss": 1.6572761535644531,
+      "eval_runtime": 51.6696,
+      "eval_samples_per_second": 19.354,
+      "eval_steps_per_second": 0.813,
+      "step": 1030
+    },
+    {
+      "epoch": 1.283136278780336,
+      "grad_norm": 2.9225106239318848,
+      "learning_rate": 3.212638376383764e-06,
+      "loss": 1.4421,
+      "step": 1031
+    },
+    {
+      "epoch": 1.2843808338518978,
+      "grad_norm": 2.9530649185180664,
+      "learning_rate": 3.210332103321033e-06,
+      "loss": 1.5138,
+      "step": 1032
+    },
+    {
+      "epoch": 1.28562538892346,
+      "grad_norm": 2.8513238430023193,
+      "learning_rate": 3.208025830258303e-06,
+      "loss": 1.411,
+      "step": 1033
+    },
+    {
+      "epoch": 1.2868699439950217,
+      "grad_norm": 2.7440171241760254,
+      "learning_rate": 3.2057195571955725e-06,
+      "loss": 1.4182,
+      "step": 1034
+    },
+    {
+      "epoch": 1.2881144990665838,
+      "grad_norm": 2.8126630783081055,
+      "learning_rate": 3.2034132841328415e-06,
+      "loss": 1.4333,
+      "step": 1035
+    },
+    {
+      "epoch": 1.2893590541381457,
+      "grad_norm": 2.8598744869232178,
+      "learning_rate": 3.201107011070111e-06,
+      "loss": 1.439,
+      "step": 1036
+    },
+    {
+      "epoch": 1.2906036092097075,
+      "grad_norm": 2.901622772216797,
+      "learning_rate": 3.1988007380073803e-06,
+      "loss": 1.4461,
+      "step": 1037
+    },
+    {
+      "epoch": 1.2918481642812694,
+      "grad_norm": 2.7670180797576904,
+      "learning_rate": 3.1964944649446493e-06,
+      "loss": 1.4772,
+      "step": 1038
+    },
+    {
+      "epoch": 1.2930927193528314,
+      "grad_norm": 2.7860350608825684,
+      "learning_rate": 3.194188191881919e-06,
+      "loss": 1.4414,
+      "step": 1039
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "grad_norm": 2.822052478790283,
+      "learning_rate": 3.1918819188191886e-06,
+      "loss": 1.4192,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "eval_loss": 1.663284420967102,
+      "eval_runtime": 47.2822,
+      "eval_samples_per_second": 21.15,
+      "eval_steps_per_second": 0.888,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2955818294959551,
+      "grad_norm": 2.909109115600586,
+      "learning_rate": 3.189575645756458e-06,
+      "loss": 1.4795,
+      "step": 1041
+    },
+    {
+      "epoch": 1.2968263845675172,
+      "grad_norm": 2.933673143386841,
+      "learning_rate": 3.187269372693727e-06,
+      "loss": 1.5038,
+      "step": 1042
+    },
+    {
+      "epoch": 1.298070939639079,
+      "grad_norm": 3.032518148422241,
+      "learning_rate": 3.1849630996309964e-06,
+      "loss": 1.5133,
+      "step": 1043
+    },
+    {
+      "epoch": 1.299315494710641,
+      "grad_norm": 2.826040506362915,
+      "learning_rate": 3.1826568265682663e-06,
+      "loss": 1.4395,
+      "step": 1044
+    },
+    {
+      "epoch": 1.3005600497822027,
+      "grad_norm": 2.8152658939361572,
+      "learning_rate": 3.1803505535055353e-06,
+      "loss": 1.4787,
+      "step": 1045
+    },
+    {
+      "epoch": 1.3018046048537648,
+      "grad_norm": 2.9410431385040283,
+      "learning_rate": 3.1780442804428047e-06,
+      "loss": 1.4375,
+      "step": 1046
+    },
+    {
+      "epoch": 1.3030491599253267,
+      "grad_norm": 2.948636531829834,
+      "learning_rate": 3.175738007380074e-06,
+      "loss": 1.4847,
+      "step": 1047
+    },
+    {
+      "epoch": 1.3042937149968887,
+      "grad_norm": 2.6287615299224854,
+      "learning_rate": 3.173431734317343e-06,
+      "loss": 1.4186,
+      "step": 1048
+    },
+    {
+      "epoch": 1.3055382700684506,
+      "grad_norm": 2.836029052734375,
+      "learning_rate": 3.1711254612546125e-06,
+      "loss": 1.4251,
+      "step": 1049
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "grad_norm": 2.982081651687622,
+      "learning_rate": 3.1688191881918824e-06,
+      "loss": 1.4951,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "eval_loss": 1.6578067541122437,
+      "eval_runtime": 47.5847,
+      "eval_samples_per_second": 21.015,
+      "eval_steps_per_second": 0.883,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3080273802115743,
+      "grad_norm": 2.9297118186950684,
+      "learning_rate": 3.166512915129152e-06,
+      "loss": 1.4699,
+      "step": 1051
+    },
+    {
+      "epoch": 1.3092719352831363,
+      "grad_norm": 2.906435251235962,
+      "learning_rate": 3.164206642066421e-06,
+      "loss": 1.4562,
+      "step": 1052
+    },
+    {
+      "epoch": 1.3105164903546982,
+      "grad_norm": 2.8655896186828613,
+      "learning_rate": 3.1619003690036902e-06,
+      "loss": 1.4351,
+      "step": 1053
+    },
+    {
+      "epoch": 1.31176104542626,
+      "grad_norm": 2.779244899749756,
+      "learning_rate": 3.15959409594096e-06,
+      "loss": 1.4479,
+      "step": 1054
+    },
+    {
+      "epoch": 1.3130056004978221,
+      "grad_norm": 2.9863739013671875,
+      "learning_rate": 3.157287822878229e-06,
+      "loss": 1.4964,
+      "step": 1055
+    },
+    {
+      "epoch": 1.314250155569384,
+      "grad_norm": 2.821871042251587,
+      "learning_rate": 3.1549815498154985e-06,
+      "loss": 1.4173,
+      "step": 1056
+    },
+    {
+      "epoch": 1.3154947106409458,
+      "grad_norm": 2.8573882579803467,
+      "learning_rate": 3.152675276752768e-06,
+      "loss": 1.4877,
+      "step": 1057
+    },
+    {
+      "epoch": 1.3167392657125077,
+      "grad_norm": 2.9824137687683105,
+      "learning_rate": 3.150369003690037e-06,
+      "loss": 1.5124,
+      "step": 1058
+    },
+    {
+      "epoch": 1.3179838207840697,
+      "grad_norm": 2.881098747253418,
+      "learning_rate": 3.1480627306273063e-06,
+      "loss": 1.4346,
+      "step": 1059
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "grad_norm": 2.918782949447632,
+      "learning_rate": 3.145756457564576e-06,
+      "loss": 1.4797,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "eval_loss": 1.6510179042816162,
+      "eval_runtime": 53.6356,
+      "eval_samples_per_second": 18.644,
+      "eval_steps_per_second": 0.783,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3204729309271936,
+      "grad_norm": 3.045776128768921,
+      "learning_rate": 3.143450184501845e-06,
+      "loss": 1.5039,
+      "step": 1061
+    },
+    {
+      "epoch": 1.3217174859987555,
+      "grad_norm": 3.041501760482788,
+      "learning_rate": 3.1411439114391146e-06,
+      "loss": 1.4878,
+      "step": 1062
+    },
+    {
+      "epoch": 1.3229620410703173,
+      "grad_norm": 2.917837142944336,
+      "learning_rate": 3.138837638376384e-06,
+      "loss": 1.4202,
+      "step": 1063
+    },
+    {
+      "epoch": 1.3242065961418792,
+      "grad_norm": 2.708070755004883,
+      "learning_rate": 3.1365313653136535e-06,
+      "loss": 1.4071,
+      "step": 1064
+    },
+    {
+      "epoch": 1.3254511512134413,
+      "grad_norm": 2.9146156311035156,
+      "learning_rate": 3.1342250922509225e-06,
+      "loss": 1.4428,
+      "step": 1065
+    },
+    {
+      "epoch": 1.326695706285003,
+      "grad_norm": 3.220919132232666,
+      "learning_rate": 3.1319188191881923e-06,
+      "loss": 1.4733,
+      "step": 1066
+    },
+    {
+      "epoch": 1.327940261356565,
+      "grad_norm": 2.9127554893493652,
+      "learning_rate": 3.1296125461254617e-06,
+      "loss": 1.4554,
+      "step": 1067
+    },
+    {
+      "epoch": 1.329184816428127,
+      "grad_norm": 3.053191661834717,
+      "learning_rate": 3.1273062730627307e-06,
+      "loss": 1.4044,
+      "step": 1068
+    },
+    {
+      "epoch": 1.3304293714996889,
+      "grad_norm": 3.2937021255493164,
+      "learning_rate": 3.125e-06,
+      "loss": 1.4949,
+      "step": 1069
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "grad_norm": 2.976717948913574,
+      "learning_rate": 3.12269372693727e-06,
+      "loss": 1.446,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "eval_loss": 1.6503105163574219,
+      "eval_runtime": 50.7579,
+      "eval_samples_per_second": 19.701,
+      "eval_steps_per_second": 0.827,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3329184816428126,
+      "grad_norm": 3.0860936641693115,
+      "learning_rate": 3.120387453874539e-06,
+      "loss": 1.4055,
+      "step": 1071
+    },
+    {
+      "epoch": 1.3341630367143746,
+      "grad_norm": 3.0401217937469482,
+      "learning_rate": 3.1180811808118084e-06,
+      "loss": 1.4688,
+      "step": 1072
+    },
+    {
+      "epoch": 1.3354075917859365,
+      "grad_norm": 2.953253984451294,
+      "learning_rate": 3.115774907749078e-06,
+      "loss": 1.5001,
+      "step": 1073
+    },
+    {
+      "epoch": 1.3366521468574986,
+      "grad_norm": 3.0665059089660645,
+      "learning_rate": 3.113468634686347e-06,
+      "loss": 1.4217,
+      "step": 1074
+    },
+    {
+      "epoch": 1.3378967019290604,
+      "grad_norm": 3.121600389480591,
+      "learning_rate": 3.1111623616236163e-06,
+      "loss": 1.4508,
+      "step": 1075
+    },
+    {
+      "epoch": 1.3391412570006223,
+      "grad_norm": 2.8057339191436768,
+      "learning_rate": 3.108856088560886e-06,
+      "loss": 1.4241,
+      "step": 1076
+    },
+    {
+      "epoch": 1.340385812072184,
+      "grad_norm": 3.0147411823272705,
+      "learning_rate": 3.1065498154981555e-06,
+      "loss": 1.479,
+      "step": 1077
+    },
+    {
+      "epoch": 1.3416303671437462,
+      "grad_norm": 3.030010223388672,
+      "learning_rate": 3.1042435424354245e-06,
+      "loss": 1.4861,
+      "step": 1078
+    },
+    {
+      "epoch": 1.342874922215308,
+      "grad_norm": 3.027998208999634,
+      "learning_rate": 3.101937269372694e-06,
+      "loss": 1.4916,
+      "step": 1079
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "grad_norm": 2.9252536296844482,
+      "learning_rate": 3.0996309963099634e-06,
+      "loss": 1.418,
+      "step": 1080
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "eval_loss": 1.6490333080291748,
+      "eval_runtime": 47.4026,
+      "eval_samples_per_second": 21.096,
+      "eval_steps_per_second": 0.886,
+      "step": 1080
+    },
+    {
+      "epoch": 1.345364032358432,
+      "grad_norm": 3.110847234725952,
+      "learning_rate": 3.0973247232472324e-06,
+      "loss": 1.4127,
+      "step": 1081
+    },
+    {
+      "epoch": 1.3466085874299938,
+      "grad_norm": 2.846491575241089,
+      "learning_rate": 3.0950184501845022e-06,
+      "loss": 1.444,
+      "step": 1082
+    },
+    {
+      "epoch": 1.3478531425015556,
+      "grad_norm": 2.950533151626587,
+      "learning_rate": 3.0927121771217716e-06,
+      "loss": 1.4456,
+      "step": 1083
+    },
+    {
+      "epoch": 1.3490976975731175,
+      "grad_norm": 2.875964403152466,
+      "learning_rate": 3.0904059040590406e-06,
+      "loss": 1.4485,
+      "step": 1084
+    },
+    {
+      "epoch": 1.3503422526446796,
+      "grad_norm": 2.9612326622009277,
+      "learning_rate": 3.08809963099631e-06,
+      "loss": 1.4446,
+      "step": 1085
+    },
+    {
+      "epoch": 1.3515868077162414,
+      "grad_norm": 2.925297975540161,
+      "learning_rate": 3.0857933579335795e-06,
+      "loss": 1.4847,
+      "step": 1086
+    },
+    {
+      "epoch": 1.3528313627878035,
+      "grad_norm": 2.991616725921631,
+      "learning_rate": 3.0834870848708485e-06,
+      "loss": 1.5151,
+      "step": 1087
+    },
+    {
+      "epoch": 1.3540759178593653,
+      "grad_norm": 2.96329927444458,
+      "learning_rate": 3.0811808118081183e-06,
+      "loss": 1.4574,
+      "step": 1088
+    },
+    {
+      "epoch": 1.3553204729309272,
+      "grad_norm": 2.948702573776245,
+      "learning_rate": 3.0788745387453878e-06,
+      "loss": 1.4585,
+      "step": 1089
+    },
+    {
+      "epoch": 1.356565028002489,
+      "grad_norm": 3.218463659286499,
+      "learning_rate": 3.076568265682657e-06,
+      "loss": 1.4439,
+      "step": 1090
+    },
+    {
+      "epoch": 1.356565028002489,
+      "eval_loss": 1.645838975906372,
+      "eval_runtime": 47.048,
+      "eval_samples_per_second": 21.255,
+      "eval_steps_per_second": 0.893,
+      "step": 1090
+    },
+    {
+      "epoch": 1.357809583074051,
+      "grad_norm": 2.9799578189849854,
+      "learning_rate": 3.074261992619926e-06,
+      "loss": 1.4883,
+      "step": 1091
+    },
+    {
+      "epoch": 1.359054138145613,
+      "grad_norm": 2.7660505771636963,
+      "learning_rate": 3.071955719557196e-06,
+      "loss": 1.4349,
+      "step": 1092
+    },
+    {
+      "epoch": 1.3602986932171748,
+      "grad_norm": 3.0201916694641113,
+      "learning_rate": 3.0696494464944655e-06,
+      "loss": 1.418,
+      "step": 1093
+    },
+    {
+      "epoch": 1.3615432482887369,
+      "grad_norm": 2.844207763671875,
+      "learning_rate": 3.0673431734317345e-06,
+      "loss": 1.4769,
+      "step": 1094
+    },
+    {
+      "epoch": 1.3627878033602987,
+      "grad_norm": 2.8306753635406494,
+      "learning_rate": 3.065036900369004e-06,
+      "loss": 1.4653,
+      "step": 1095
+    },
+    {
+      "epoch": 1.3640323584318605,
+      "grad_norm": 2.805023431777954,
+      "learning_rate": 3.0627306273062733e-06,
+      "loss": 1.4771,
+      "step": 1096
+    },
+    {
+      "epoch": 1.3652769135034224,
+      "grad_norm": 3.020883083343506,
+      "learning_rate": 3.0604243542435423e-06,
+      "loss": 1.4237,
+      "step": 1097
+    },
+    {
+      "epoch": 1.3665214685749845,
+      "grad_norm": 2.8395731449127197,
+      "learning_rate": 3.058118081180812e-06,
+      "loss": 1.4647,
+      "step": 1098
+    },
+    {
+      "epoch": 1.3677660236465463,
+      "grad_norm": 2.871156692504883,
+      "learning_rate": 3.0558118081180816e-06,
+      "loss": 1.4196,
+      "step": 1099
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "grad_norm": 2.974905014038086,
+      "learning_rate": 3.053505535055351e-06,
+      "loss": 1.4779,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "eval_loss": 1.651503562927246,
+      "eval_runtime": 54.2713,
+      "eval_samples_per_second": 18.426,
+      "eval_steps_per_second": 0.774,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3702551337896702,
+      "grad_norm": 2.7591328620910645,
+      "learning_rate": 3.05119926199262e-06,
+      "loss": 1.3899,
+      "step": 1101
+    },
+    {
+      "epoch": 1.371499688861232,
+      "grad_norm": 2.8575668334960938,
+      "learning_rate": 3.0488929889298894e-06,
+      "loss": 1.4482,
+      "step": 1102
+    },
+    {
+      "epoch": 1.372744243932794,
+      "grad_norm": 2.9049339294433594,
+      "learning_rate": 3.0465867158671593e-06,
+      "loss": 1.4785,
+      "step": 1103
+    },
+    {
+      "epoch": 1.373988799004356,
+      "grad_norm": 2.816453695297241,
+      "learning_rate": 3.0442804428044283e-06,
+      "loss": 1.4601,
+      "step": 1104
+    },
+    {
+      "epoch": 1.3752333540759178,
+      "grad_norm": 2.8844189643859863,
+      "learning_rate": 3.0419741697416977e-06,
+      "loss": 1.496,
+      "step": 1105
+    },
+    {
+      "epoch": 1.3764779091474797,
+      "grad_norm": 3.014744281768799,
+      "learning_rate": 3.039667896678967e-06,
+      "loss": 1.4098,
+      "step": 1106
+    },
+    {
+      "epoch": 1.3777224642190418,
+      "grad_norm": 2.8243560791015625,
+      "learning_rate": 3.037361623616236e-06,
+      "loss": 1.4086,
+      "step": 1107
+    },
+    {
+      "epoch": 1.3789670192906036,
+      "grad_norm": 2.8333253860473633,
+      "learning_rate": 3.035055350553506e-06,
+      "loss": 1.3516,
+      "step": 1108
+    },
+    {
+      "epoch": 1.3802115743621655,
+      "grad_norm": 2.926455020904541,
+      "learning_rate": 3.0327490774907754e-06,
+      "loss": 1.3976,
+      "step": 1109
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "grad_norm": 2.900937080383301,
+      "learning_rate": 3.0304428044280444e-06,
+      "loss": 1.4336,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "eval_loss": 1.647048830986023,
+      "eval_runtime": 42.3955,
+      "eval_samples_per_second": 23.587,
+      "eval_steps_per_second": 0.991,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3827006845052894,
+      "grad_norm": 3.0756969451904297,
+      "learning_rate": 3.028136531365314e-06,
+      "loss": 1.4674,
+      "step": 1111
+    },
+    {
+      "epoch": 1.3839452395768512,
+      "grad_norm": 2.7754578590393066,
+      "learning_rate": 3.0258302583025832e-06,
+      "loss": 1.4435,
+      "step": 1112
+    },
+    {
+      "epoch": 1.3851897946484133,
+      "grad_norm": 2.8011393547058105,
+      "learning_rate": 3.023523985239853e-06,
+      "loss": 1.4212,
+      "step": 1113
+    },
+    {
+      "epoch": 1.3864343497199751,
+      "grad_norm": 2.7063710689544678,
+      "learning_rate": 3.021217712177122e-06,
+      "loss": 1.3769,
+      "step": 1114
+    },
+    {
+      "epoch": 1.387678904791537,
+      "grad_norm": 2.830411434173584,
+      "learning_rate": 3.0189114391143915e-06,
+      "loss": 1.4832,
+      "step": 1115
+    },
+    {
+      "epoch": 1.3889234598630988,
+      "grad_norm": 2.9075911045074463,
+      "learning_rate": 3.016605166051661e-06,
+      "loss": 1.486,
+      "step": 1116
+    },
+    {
+      "epoch": 1.390168014934661,
+      "grad_norm": 2.847822666168213,
+      "learning_rate": 3.01429889298893e-06,
+      "loss": 1.424,
+      "step": 1117
+    },
+    {
+      "epoch": 1.3914125700062228,
+      "grad_norm": 2.8822033405303955,
+      "learning_rate": 3.0119926199261993e-06,
+      "loss": 1.4218,
+      "step": 1118
+    },
+    {
+      "epoch": 1.3926571250777848,
+      "grad_norm": 2.8944740295410156,
+      "learning_rate": 3.009686346863469e-06,
+      "loss": 1.4283,
+      "step": 1119
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "grad_norm": 2.765190839767456,
+      "learning_rate": 3.007380073800738e-06,
+      "loss": 1.4622,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "eval_loss": 1.6451724767684937,
+      "eval_runtime": 43.7449,
+      "eval_samples_per_second": 22.86,
+      "eval_steps_per_second": 0.96,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3951462352209085,
+      "grad_norm": 2.877002716064453,
+      "learning_rate": 3.0050738007380076e-06,
+      "loss": 1.4145,
+      "step": 1121
+    },
+    {
+      "epoch": 1.3963907902924704,
+      "grad_norm": 2.870389699935913,
+      "learning_rate": 3.002767527675277e-06,
+      "loss": 1.4401,
+      "step": 1122
+    },
+    {
+      "epoch": 1.3976353453640322,
+      "grad_norm": 2.7788920402526855,
+      "learning_rate": 3.000461254612546e-06,
+      "loss": 1.4514,
+      "step": 1123
+    },
+    {
+      "epoch": 1.3988799004355943,
+      "grad_norm": 2.9269332885742188,
+      "learning_rate": 2.998154981549816e-06,
+      "loss": 1.3961,
+      "step": 1124
+    },
+    {
+      "epoch": 1.4001244555071561,
+      "grad_norm": 3.013291835784912,
+      "learning_rate": 2.9958487084870853e-06,
+      "loss": 1.4597,
+      "step": 1125
+    },
+    {
+      "epoch": 1.4013690105787182,
+      "grad_norm": 2.9063880443573,
+      "learning_rate": 2.9935424354243547e-06,
+      "loss": 1.4668,
+      "step": 1126
+    },
+    {
+      "epoch": 1.40261356565028,
+      "grad_norm": 2.8926970958709717,
+      "learning_rate": 2.9912361623616237e-06,
+      "loss": 1.4332,
+      "step": 1127
+    },
+    {
+      "epoch": 1.403858120721842,
+      "grad_norm": 2.8563647270202637,
+      "learning_rate": 2.988929889298893e-06,
+      "loss": 1.407,
+      "step": 1128
+    },
+    {
+      "epoch": 1.4051026757934038,
+      "grad_norm": 2.8725359439849854,
+      "learning_rate": 2.986623616236163e-06,
+      "loss": 1.4434,
+      "step": 1129
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "grad_norm": 2.906898260116577,
+      "learning_rate": 2.984317343173432e-06,
+      "loss": 1.4745,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "eval_loss": 1.647884726524353,
+      "eval_runtime": 44.9221,
+      "eval_samples_per_second": 22.261,
+      "eval_steps_per_second": 0.935,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4075917859365277,
+      "grad_norm": 2.8297719955444336,
+      "learning_rate": 2.9820110701107014e-06,
+      "loss": 1.4542,
+      "step": 1131
+    },
+    {
+      "epoch": 1.4088363410080897,
+      "grad_norm": 2.841031551361084,
+      "learning_rate": 2.979704797047971e-06,
+      "loss": 1.4105,
+      "step": 1132
+    },
+    {
+      "epoch": 1.4100808960796516,
+      "grad_norm": 2.867791175842285,
+      "learning_rate": 2.97739852398524e-06,
+      "loss": 1.4409,
+      "step": 1133
+    },
+    {
+      "epoch": 1.4113254511512134,
+      "grad_norm": 2.8156039714813232,
+      "learning_rate": 2.9750922509225093e-06,
+      "loss": 1.4284,
+      "step": 1134
+    },
+    {
+      "epoch": 1.4125700062227753,
+      "grad_norm": 2.825985908508301,
+      "learning_rate": 2.972785977859779e-06,
+      "loss": 1.4594,
+      "step": 1135
+    },
+    {
+      "epoch": 1.4138145612943374,
+      "grad_norm": 3.0134711265563965,
+      "learning_rate": 2.970479704797048e-06,
+      "loss": 1.4743,
+      "step": 1136
+    },
+    {
+      "epoch": 1.4150591163658992,
+      "grad_norm": 2.9829142093658447,
+      "learning_rate": 2.9681734317343175e-06,
+      "loss": 1.4935,
+      "step": 1137
+    },
+    {
+      "epoch": 1.416303671437461,
+      "grad_norm": 3.067620277404785,
+      "learning_rate": 2.965867158671587e-06,
+      "loss": 1.4266,
+      "step": 1138
+    },
+    {
+      "epoch": 1.4175482265090231,
+      "grad_norm": 2.8969919681549072,
+      "learning_rate": 2.9635608856088564e-06,
+      "loss": 1.451,
+      "step": 1139
+    },
+    {
+      "epoch": 1.418792781580585,
+      "grad_norm": 2.8706412315368652,
+      "learning_rate": 2.9612546125461254e-06,
+      "loss": 1.4834,
+      "step": 1140
+    },
+    {
+      "epoch": 1.418792781580585,
+      "eval_loss": 1.643480658531189,
+      "eval_runtime": 44.743,
+      "eval_samples_per_second": 22.35,
+      "eval_steps_per_second": 0.939,
+      "step": 1140
+    },
+    {
+      "epoch": 1.4200373366521468,
+      "grad_norm": 2.8801848888397217,
+      "learning_rate": 2.958948339483395e-06,
+      "loss": 1.4258,
+      "step": 1141
+    },
+    {
+      "epoch": 1.4212818917237087,
+      "grad_norm": 2.774635076522827,
+      "learning_rate": 2.9566420664206646e-06,
+      "loss": 1.416,
+      "step": 1142
+    },
+    {
+      "epoch": 1.4225264467952707,
+      "grad_norm": 2.8460140228271484,
+      "learning_rate": 2.9543357933579336e-06,
+      "loss": 1.4213,
+      "step": 1143
+    },
+    {
+      "epoch": 1.4237710018668326,
+      "grad_norm": 2.907888889312744,
+      "learning_rate": 2.952029520295203e-06,
+      "loss": 1.4507,
+      "step": 1144
+    },
+    {
+      "epoch": 1.4250155569383947,
+      "grad_norm": 2.789041757583618,
+      "learning_rate": 2.949723247232473e-06,
+      "loss": 1.4257,
+      "step": 1145
+    },
+    {
+      "epoch": 1.4262601120099565,
+      "grad_norm": 2.78859806060791,
+      "learning_rate": 2.947416974169742e-06,
+      "loss": 1.4377,
+      "step": 1146
+    },
+    {
+      "epoch": 1.4275046670815184,
+      "grad_norm": 2.840303421020508,
+      "learning_rate": 2.9451107011070113e-06,
+      "loss": 1.4684,
+      "step": 1147
+    },
+    {
+      "epoch": 1.4287492221530802,
+      "grad_norm": 2.8800530433654785,
+      "learning_rate": 2.9428044280442807e-06,
+      "loss": 1.4912,
+      "step": 1148
+    },
+    {
+      "epoch": 1.4299937772246423,
+      "grad_norm": 2.841092824935913,
+      "learning_rate": 2.9404981549815497e-06,
+      "loss": 1.4234,
+      "step": 1149
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "grad_norm": 2.8226537704467773,
+      "learning_rate": 2.938191881918819e-06,
+      "loss": 1.4578,
+      "step": 1150
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "eval_loss": 1.643249273300171,
+      "eval_runtime": 43.5586,
+      "eval_samples_per_second": 22.958,
+      "eval_steps_per_second": 0.964,
+      "step": 1150
+    },
+    {
+      "epoch": 1.432482887367766,
+      "grad_norm": 2.869736909866333,
+      "learning_rate": 2.935885608856089e-06,
+      "loss": 1.4701,
+      "step": 1151
+    },
+    {
+      "epoch": 1.433727442439328,
+      "grad_norm": 2.689211130142212,
+      "learning_rate": 2.9335793357933584e-06,
+      "loss": 1.4483,
+      "step": 1152
+    },
+    {
+      "epoch": 1.4349719975108899,
+      "grad_norm": 2.9267077445983887,
+      "learning_rate": 2.9312730627306274e-06,
+      "loss": 1.4446,
+      "step": 1153
+    },
+    {
+      "epoch": 1.4362165525824517,
+      "grad_norm": 2.906862497329712,
+      "learning_rate": 2.928966789667897e-06,
+      "loss": 1.4524,
+      "step": 1154
+    },
+    {
+      "epoch": 1.4374611076540136,
+      "grad_norm": 2.8249800205230713,
+      "learning_rate": 2.9266605166051663e-06,
+      "loss": 1.4208,
+      "step": 1155
+    },
+    {
+      "epoch": 1.4387056627255757,
+      "grad_norm": 2.9836032390594482,
+      "learning_rate": 2.9243542435424353e-06,
+      "loss": 1.4581,
+      "step": 1156
+    },
+    {
+      "epoch": 1.4399502177971375,
+      "grad_norm": 2.9595284461975098,
+      "learning_rate": 2.922047970479705e-06,
+      "loss": 1.4141,
+      "step": 1157
+    },
+    {
+      "epoch": 1.4411947728686996,
+      "grad_norm": 3.0363423824310303,
+      "learning_rate": 2.9197416974169746e-06,
+      "loss": 1.4859,
+      "step": 1158
+    },
+    {
+      "epoch": 1.4424393279402614,
+      "grad_norm": 2.911201000213623,
+      "learning_rate": 2.9174354243542436e-06,
+      "loss": 1.4346,
+      "step": 1159
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "grad_norm": 2.8617193698883057,
+      "learning_rate": 2.915129151291513e-06,
+      "loss": 1.433,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "eval_loss": 1.6437232494354248,
+      "eval_runtime": 43.961,
+      "eval_samples_per_second": 22.747,
+      "eval_steps_per_second": 0.955,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4449284380833851,
+      "grad_norm": 2.9584295749664307,
+      "learning_rate": 2.912822878228783e-06,
+      "loss": 1.4847,
+      "step": 1161
+    },
+    {
+      "epoch": 1.4461729931549472,
+      "grad_norm": 2.8209400177001953,
+      "learning_rate": 2.9105166051660522e-06,
+      "loss": 1.421,
+      "step": 1162
+    },
+    {
+      "epoch": 1.447417548226509,
+      "grad_norm": 2.847637414932251,
+      "learning_rate": 2.9082103321033212e-06,
+      "loss": 1.4326,
+      "step": 1163
+    },
+    {
+      "epoch": 1.4486621032980709,
+      "grad_norm": 2.836228132247925,
+      "learning_rate": 2.9059040590405907e-06,
+      "loss": 1.4459,
+      "step": 1164
+    },
+    {
+      "epoch": 1.449906658369633,
+      "grad_norm": 2.8327207565307617,
+      "learning_rate": 2.90359778597786e-06,
+      "loss": 1.4189,
+      "step": 1165
+    },
+    {
+      "epoch": 1.4511512134411948,
+      "grad_norm": 2.880643606185913,
+      "learning_rate": 2.901291512915129e-06,
+      "loss": 1.4214,
+      "step": 1166
+    },
+    {
+      "epoch": 1.4523957685127566,
+      "grad_norm": 2.9637348651885986,
+      "learning_rate": 2.898985239852399e-06,
+      "loss": 1.4471,
+      "step": 1167
+    },
+    {
+      "epoch": 1.4536403235843185,
+      "grad_norm": 3.006145477294922,
+      "learning_rate": 2.8966789667896684e-06,
+      "loss": 1.4228,
+      "step": 1168
+    },
+    {
+      "epoch": 1.4548848786558806,
+      "grad_norm": 2.8547627925872803,
+      "learning_rate": 2.8943726937269374e-06,
+      "loss": 1.4707,
+      "step": 1169
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "grad_norm": 2.973092794418335,
+      "learning_rate": 2.8920664206642068e-06,
+      "loss": 1.4804,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "eval_loss": 1.6340677738189697,
+      "eval_runtime": 44.6094,
+      "eval_samples_per_second": 22.417,
+      "eval_steps_per_second": 0.942,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4573739887990045,
+      "grad_norm": 2.9433603286743164,
+      "learning_rate": 2.889760147601476e-06,
+      "loss": 1.4354,
+      "step": 1171
+    },
+    {
+      "epoch": 1.4586185438705663,
+      "grad_norm": 2.872236728668213,
+      "learning_rate": 2.887453874538745e-06,
+      "loss": 1.418,
+      "step": 1172
+    },
+    {
+      "epoch": 1.4598630989421282,
+      "grad_norm": 2.847557783126831,
+      "learning_rate": 2.885147601476015e-06,
+      "loss": 1.4306,
+      "step": 1173
+    },
+    {
+      "epoch": 1.46110765401369,
+      "grad_norm": 2.7554969787597656,
+      "learning_rate": 2.8828413284132845e-06,
+      "loss": 1.4295,
+      "step": 1174
+    },
+    {
+      "epoch": 1.462352209085252,
+      "grad_norm": 2.8754842281341553,
+      "learning_rate": 2.880535055350554e-06,
+      "loss": 1.4482,
+      "step": 1175
+    },
+    {
+      "epoch": 1.463596764156814,
+      "grad_norm": 3.0115292072296143,
+      "learning_rate": 2.878228782287823e-06,
+      "loss": 1.4428,
+      "step": 1176
+    },
+    {
+      "epoch": 1.4648413192283758,
+      "grad_norm": 2.8976168632507324,
+      "learning_rate": 2.8759225092250923e-06,
+      "loss": 1.4707,
+      "step": 1177
+    },
+    {
+      "epoch": 1.4660858742999379,
+      "grad_norm": 2.7981812953948975,
+      "learning_rate": 2.873616236162362e-06,
+      "loss": 1.4693,
+      "step": 1178
+    },
+    {
+      "epoch": 1.4673304293714997,
+      "grad_norm": 2.8562371730804443,
+      "learning_rate": 2.871309963099631e-06,
+      "loss": 1.4242,
+      "step": 1179
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "grad_norm": 2.8705570697784424,
+      "learning_rate": 2.8690036900369006e-06,
+      "loss": 1.4725,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "eval_loss": 1.6338286399841309,
+      "eval_runtime": 44.7171,
+      "eval_samples_per_second": 22.363,
+      "eval_steps_per_second": 0.939,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4698195395146234,
+      "grad_norm": 2.8025710582733154,
+      "learning_rate": 2.86669741697417e-06,
+      "loss": 1.427,
+      "step": 1181
+    },
+    {
+      "epoch": 1.4710640945861855,
+      "grad_norm": 2.845151901245117,
+      "learning_rate": 2.864391143911439e-06,
+      "loss": 1.4402,
+      "step": 1182
+    },
+    {
+      "epoch": 1.4723086496577473,
+      "grad_norm": 2.943373680114746,
+      "learning_rate": 2.862084870848709e-06,
+      "loss": 1.5194,
+      "step": 1183
+    },
+    {
+      "epoch": 1.4735532047293094,
+      "grad_norm": 2.896955966949463,
+      "learning_rate": 2.8597785977859783e-06,
+      "loss": 1.4242,
+      "step": 1184
+    },
+    {
+      "epoch": 1.4747977598008712,
+      "grad_norm": 3.131305456161499,
+      "learning_rate": 2.8574723247232473e-06,
+      "loss": 1.5008,
+      "step": 1185
+    },
+    {
+      "epoch": 1.476042314872433,
+      "grad_norm": 2.8605735301971436,
+      "learning_rate": 2.8551660516605167e-06,
+      "loss": 1.3533,
+      "step": 1186
+    },
+    {
+      "epoch": 1.477286869943995,
+      "grad_norm": 2.9158811569213867,
+      "learning_rate": 2.852859778597786e-06,
+      "loss": 1.4404,
+      "step": 1187
+    },
+    {
+      "epoch": 1.478531425015557,
+      "grad_norm": 2.7795321941375732,
+      "learning_rate": 2.850553505535056e-06,
+      "loss": 1.3977,
+      "step": 1188
+    },
+    {
+      "epoch": 1.4797759800871189,
+      "grad_norm": 2.8194212913513184,
+      "learning_rate": 2.848247232472325e-06,
+      "loss": 1.4111,
+      "step": 1189
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "grad_norm": 2.983750104904175,
+      "learning_rate": 2.8459409594095944e-06,
+      "loss": 1.4061,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "eval_loss": 1.6375409364700317,
+      "eval_runtime": 45.4026,
+      "eval_samples_per_second": 22.025,
+      "eval_steps_per_second": 0.925,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4822650902302428,
+      "grad_norm": 3.0246665477752686,
+      "learning_rate": 2.843634686346864e-06,
+      "loss": 1.4532,
+      "step": 1191
+    },
+    {
+      "epoch": 1.4835096453018046,
+      "grad_norm": 2.8856449127197266,
+      "learning_rate": 2.841328413284133e-06,
+      "loss": 1.4528,
+      "step": 1192
+    },
+    {
+      "epoch": 1.4847542003733665,
+      "grad_norm": 2.848987102508545,
+      "learning_rate": 2.8390221402214022e-06,
+      "loss": 1.4369,
+      "step": 1193
+    },
+    {
+      "epoch": 1.4859987554449283,
+      "grad_norm": 2.970419406890869,
+      "learning_rate": 2.836715867158672e-06,
+      "loss": 1.4605,
+      "step": 1194
+    },
+    {
+      "epoch": 1.4872433105164904,
+      "grad_norm": 3.156613826751709,
+      "learning_rate": 2.834409594095941e-06,
+      "loss": 1.4489,
+      "step": 1195
+    },
+    {
+      "epoch": 1.4884878655880522,
+      "grad_norm": 3.095349073410034,
+      "learning_rate": 2.8321033210332105e-06,
+      "loss": 1.4604,
+      "step": 1196
+    },
+    {
+      "epoch": 1.4897324206596143,
+      "grad_norm": 2.870742082595825,
+      "learning_rate": 2.82979704797048e-06,
+      "loss": 1.4284,
+      "step": 1197
+    },
+    {
+      "epoch": 1.4909769757311762,
+      "grad_norm": 2.9415056705474854,
+      "learning_rate": 2.827490774907749e-06,
+      "loss": 1.4715,
+      "step": 1198
+    },
+    {
+      "epoch": 1.492221530802738,
+      "grad_norm": 2.8297767639160156,
+      "learning_rate": 2.8251845018450188e-06,
+      "loss": 1.4508,
+      "step": 1199
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "grad_norm": 2.872086524963379,
+      "learning_rate": 2.822878228782288e-06,
+      "loss": 1.5098,
+      "step": 1200
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "eval_loss": 1.6357526779174805,
+      "eval_runtime": 51.422,
+      "eval_samples_per_second": 19.447,
+      "eval_steps_per_second": 0.817,
+      "step": 1200
+    },
+    {
+      "epoch": 1.494710640945862,
+      "grad_norm": 2.7152082920074463,
+      "learning_rate": 2.8205719557195576e-06,
+      "loss": 1.4329,
+      "step": 1201
+    },
+    {
+      "epoch": 1.4959551960174238,
+      "grad_norm": 2.866170883178711,
+      "learning_rate": 2.8182656826568266e-06,
+      "loss": 1.4455,
+      "step": 1202
+    },
+    {
+      "epoch": 1.4971997510889856,
+      "grad_norm": 2.9010684490203857,
+      "learning_rate": 2.815959409594096e-06,
+      "loss": 1.423,
+      "step": 1203
+    },
+    {
+      "epoch": 1.4984443061605477,
+      "grad_norm": 2.8588502407073975,
+      "learning_rate": 2.813653136531366e-06,
+      "loss": 1.4041,
+      "step": 1204
+    },
+    {
+      "epoch": 1.4996888612321095,
+      "grad_norm": 2.932544231414795,
+      "learning_rate": 2.811346863468635e-06,
+      "loss": 1.4559,
+      "step": 1205
+    },
+    {
+      "epoch": 1.5009334163036714,
+      "grad_norm": 2.7486014366149902,
+      "learning_rate": 2.8090405904059043e-06,
+      "loss": 1.4337,
+      "step": 1206
+    },
+    {
+      "epoch": 1.5021779713752332,
+      "grad_norm": 3.0866589546203613,
+      "learning_rate": 2.8067343173431737e-06,
+      "loss": 1.4186,
+      "step": 1207
+    },
+    {
+      "epoch": 1.5034225264467953,
+      "grad_norm": 2.9710276126861572,
+      "learning_rate": 2.8044280442804427e-06,
+      "loss": 1.419,
+      "step": 1208
+    },
+    {
+      "epoch": 1.5046670815183572,
+      "grad_norm": 2.8379950523376465,
+      "learning_rate": 2.802121771217712e-06,
+      "loss": 1.4922,
+      "step": 1209
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "grad_norm": 2.939629554748535,
+      "learning_rate": 2.799815498154982e-06,
+      "loss": 1.4402,
+      "step": 1210
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "eval_loss": 1.6419442892074585,
+      "eval_runtime": 43.1202,
+      "eval_samples_per_second": 23.191,
+      "eval_steps_per_second": 0.974,
+      "step": 1210
+    },
+    {
+      "epoch": 1.507156191661481,
+      "grad_norm": 3.041389226913452,
+      "learning_rate": 2.7975092250922514e-06,
+      "loss": 1.4873,
+      "step": 1211
+    },
+    {
+      "epoch": 1.508400746733043,
+      "grad_norm": 2.9776995182037354,
+      "learning_rate": 2.7952029520295204e-06,
+      "loss": 1.4422,
+      "step": 1212
+    },
+    {
+      "epoch": 1.5096453018046048,
+      "grad_norm": 2.8798792362213135,
+      "learning_rate": 2.79289667896679e-06,
+      "loss": 1.4535,
+      "step": 1213
+    },
+    {
+      "epoch": 1.5108898568761666,
+      "grad_norm": 2.9016385078430176,
+      "learning_rate": 2.7905904059040597e-06,
+      "loss": 1.4462,
+      "step": 1214
+    },
+    {
+      "epoch": 1.5121344119477287,
+      "grad_norm": 2.8987581729888916,
+      "learning_rate": 2.7882841328413287e-06,
+      "loss": 1.4477,
+      "step": 1215
+    },
+    {
+      "epoch": 1.5133789670192908,
+      "grad_norm": 2.960266351699829,
+      "learning_rate": 2.785977859778598e-06,
+      "loss": 1.4811,
+      "step": 1216
+    },
+    {
+      "epoch": 1.5146235220908526,
+      "grad_norm": 2.8965611457824707,
+      "learning_rate": 2.7836715867158675e-06,
+      "loss": 1.4366,
+      "step": 1217
+    },
+    {
+      "epoch": 1.5158680771624145,
+      "grad_norm": 3.0667364597320557,
+      "learning_rate": 2.7813653136531365e-06,
+      "loss": 1.3828,
+      "step": 1218
+    },
+    {
+      "epoch": 1.5171126322339763,
+      "grad_norm": 2.952362537384033,
+      "learning_rate": 2.779059040590406e-06,
+      "loss": 1.3903,
+      "step": 1219
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "grad_norm": 2.9899795055389404,
+      "learning_rate": 2.776752767527676e-06,
+      "loss": 1.4312,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "eval_loss": 1.640383005142212,
+      "eval_runtime": 45.4128,
+      "eval_samples_per_second": 22.02,
+      "eval_steps_per_second": 0.925,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5196017423771002,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4344,
+      "step": 1221
+    },
+    {
+      "epoch": 1.520846297448662,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4386,
+      "step": 1222
+    },
+    {
+      "epoch": 1.5220908525202241,
+      "grad_norm": 2.883223295211792,
+      "learning_rate": 2.7721402214022142e-06,
+      "loss": 1.4401,
+      "step": 1223
+    },
+    {
+      "epoch": 1.523335407591786,
+      "grad_norm": 2.8346424102783203,
+      "learning_rate": 2.7698339483394837e-06,
+      "loss": 1.3926,
+      "step": 1224
+    },
+    {
+      "epoch": 1.5245799626633478,
+      "grad_norm": 2.8898870944976807,
+      "learning_rate": 2.767527675276753e-06,
+      "loss": 1.4128,
+      "step": 1225
+    },
+    {
+      "epoch": 1.5258245177349097,
+      "grad_norm": 2.828594923019409,
+      "learning_rate": 2.765221402214022e-06,
+      "loss": 1.4103,
+      "step": 1226
+    },
+    {
+      "epoch": 1.5270690728064715,
+      "grad_norm": 2.882558822631836,
+      "learning_rate": 2.762915129151292e-06,
+      "loss": 1.4186,
+      "step": 1227
+    },
+    {
+      "epoch": 1.5283136278780336,
+      "grad_norm": 2.955760955810547,
+      "learning_rate": 2.7606088560885613e-06,
+      "loss": 1.3705,
+      "step": 1228
+    },
+    {
+      "epoch": 1.5295581829495957,
+      "grad_norm": 2.862934112548828,
+      "learning_rate": 2.7583025830258303e-06,
+      "loss": 1.5012,
+      "step": 1229
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "grad_norm": 2.926501989364624,
+      "learning_rate": 2.7559963099630998e-06,
+      "loss": 1.3875,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "eval_loss": 1.6376254558563232,
+      "eval_runtime": 47.9535,
+      "eval_samples_per_second": 20.854,
+      "eval_steps_per_second": 0.876,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5320472930927194,
+      "grad_norm": 2.7648823261260986,
+      "learning_rate": 2.753690036900369e-06,
+      "loss": 1.3781,
+      "step": 1231
+    },
+    {
+      "epoch": 1.5332918481642812,
+      "grad_norm": 2.927103281021118,
+      "learning_rate": 2.751383763837638e-06,
+      "loss": 1.4091,
+      "step": 1232
+    },
+    {
+      "epoch": 1.534536403235843,
+      "grad_norm": 2.964823007583618,
+      "learning_rate": 2.749077490774908e-06,
+      "loss": 1.5075,
+      "step": 1233
+    },
+    {
+      "epoch": 1.5357809583074051,
+      "grad_norm": 2.7944016456604004,
+      "learning_rate": 2.7467712177121775e-06,
+      "loss": 1.3888,
+      "step": 1234
+    },
+    {
+      "epoch": 1.537025513378967,
+      "grad_norm": 2.891592025756836,
+      "learning_rate": 2.7444649446494465e-06,
+      "loss": 1.4825,
+      "step": 1235
+    },
+    {
+      "epoch": 1.538270068450529,
+      "grad_norm": 2.8120031356811523,
+      "learning_rate": 2.742158671586716e-06,
+      "loss": 1.4246,
+      "step": 1236
+    },
+    {
+      "epoch": 1.539514623522091,
+      "grad_norm": 2.9384255409240723,
+      "learning_rate": 2.7398523985239857e-06,
+      "loss": 1.4657,
+      "step": 1237
+    },
+    {
+      "epoch": 1.5407591785936527,
+      "grad_norm": 2.73111891746521,
+      "learning_rate": 2.737546125461255e-06,
+      "loss": 1.4689,
+      "step": 1238
+    },
+    {
+      "epoch": 1.5420037336652146,
+      "grad_norm": 2.858963966369629,
+      "learning_rate": 2.735239852398524e-06,
+      "loss": 1.4626,
+      "step": 1239
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "grad_norm": 2.9148507118225098,
+      "learning_rate": 2.7329335793357936e-06,
+      "loss": 1.4787,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "eval_loss": 1.6436412334442139,
+      "eval_runtime": 47.3139,
+      "eval_samples_per_second": 21.135,
+      "eval_steps_per_second": 0.888,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5444928438083385,
+      "grad_norm": 2.8731327056884766,
+      "learning_rate": 2.730627306273063e-06,
+      "loss": 1.4748,
+      "step": 1241
+    },
+    {
+      "epoch": 1.5457373988799006,
+      "grad_norm": 2.990265369415283,
+      "learning_rate": 2.728321033210332e-06,
+      "loss": 1.4375,
+      "step": 1242
+    },
+    {
+      "epoch": 1.5469819539514624,
+      "grad_norm": 2.874321460723877,
+      "learning_rate": 2.726014760147602e-06,
+      "loss": 1.4154,
+      "step": 1243
+    },
+    {
+      "epoch": 1.5482265090230243,
+      "grad_norm": 2.9316225051879883,
+      "learning_rate": 2.7237084870848713e-06,
+      "loss": 1.4285,
+      "step": 1244
+    },
+    {
+      "epoch": 1.5494710640945861,
+      "grad_norm": 2.783966302871704,
+      "learning_rate": 2.7214022140221403e-06,
+      "loss": 1.409,
+      "step": 1245
+    },
+    {
+      "epoch": 1.550715619166148,
+      "grad_norm": 2.9578707218170166,
+      "learning_rate": 2.7190959409594097e-06,
+      "loss": 1.4325,
+      "step": 1246
+    },
+    {
+      "epoch": 1.55196017423771,
+      "grad_norm": 2.869893789291382,
+      "learning_rate": 2.716789667896679e-06,
+      "loss": 1.3989,
+      "step": 1247
+    },
+    {
+      "epoch": 1.553204729309272,
+      "grad_norm": 2.962944269180298,
+      "learning_rate": 2.714483394833948e-06,
+      "loss": 1.4302,
+      "step": 1248
+    },
+    {
+      "epoch": 1.554449284380834,
+      "grad_norm": 2.974494695663452,
+      "learning_rate": 2.712177121771218e-06,
+      "loss": 1.46,
+      "step": 1249
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "grad_norm": 2.9154603481292725,
+      "learning_rate": 2.7098708487084874e-06,
+      "loss": 1.4388,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "eval_loss": 1.6357773542404175,
+      "eval_runtime": 47.6085,
+      "eval_samples_per_second": 21.005,
+      "eval_steps_per_second": 0.882,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5569383945239577,
+      "grad_norm": 3.0333547592163086,
+      "learning_rate": 2.707564575645757e-06,
+      "loss": 1.4846,
+      "step": 1251
+    },
+    {
+      "epoch": 1.5581829495955195,
+      "grad_norm": 3.06097674369812,
+      "learning_rate": 2.705258302583026e-06,
+      "loss": 1.4355,
+      "step": 1252
+    },
+    {
+      "epoch": 1.5594275046670814,
+      "grad_norm": 2.872680425643921,
+      "learning_rate": 2.7029520295202956e-06,
+      "loss": 1.4219,
+      "step": 1253
+    },
+    {
+      "epoch": 1.5606720597386434,
+      "grad_norm": 2.9066731929779053,
+      "learning_rate": 2.700645756457565e-06,
+      "loss": 1.384,
+      "step": 1254
+    },
+    {
+      "epoch": 1.5619166148102055,
+      "grad_norm": 3.0570194721221924,
+      "learning_rate": 2.698339483394834e-06,
+      "loss": 1.4687,
+      "step": 1255
+    },
+    {
+      "epoch": 1.5631611698817673,
+      "grad_norm": 3.1955857276916504,
+      "learning_rate": 2.6960332103321035e-06,
+      "loss": 1.4736,
+      "step": 1256
+    },
+    {
+      "epoch": 1.5644057249533292,
+      "grad_norm": 2.9686198234558105,
+      "learning_rate": 2.693726937269373e-06,
+      "loss": 1.4176,
+      "step": 1257
+    },
+    {
+      "epoch": 1.565650280024891,
+      "grad_norm": 3.1161272525787354,
+      "learning_rate": 2.691420664206642e-06,
+      "loss": 1.443,
+      "step": 1258
+    },
+    {
+      "epoch": 1.5668948350964529,
+      "grad_norm": 2.9031193256378174,
+      "learning_rate": 2.6891143911439118e-06,
+      "loss": 1.3871,
+      "step": 1259
+    },
+    {
+      "epoch": 1.568139390168015,
+      "grad_norm": 2.9120476245880127,
+      "learning_rate": 2.686808118081181e-06,
+      "loss": 1.4406,
+      "step": 1260
+    },
+    {
+      "epoch": 1.568139390168015,
+      "eval_loss": 1.6291446685791016,
+      "eval_runtime": 49.7006,
+      "eval_samples_per_second": 20.12,
+      "eval_steps_per_second": 0.845,
+      "step": 1260
+    },
+    {
+      "epoch": 1.5693839452395768,
+      "grad_norm": 3.3181824684143066,
+      "learning_rate": 2.6845018450184506e-06,
+      "loss": 1.5278,
+      "step": 1261
+    },
+    {
+      "epoch": 1.5706285003111389,
+      "grad_norm": 2.9628148078918457,
+      "learning_rate": 2.6821955719557196e-06,
+      "loss": 1.4206,
+      "step": 1262
+    },
+    {
+      "epoch": 1.5718730553827007,
+      "grad_norm": 2.9828474521636963,
+      "learning_rate": 2.679889298892989e-06,
+      "loss": 1.5478,
+      "step": 1263
+    },
+    {
+      "epoch": 1.5731176104542626,
+      "grad_norm": 3.018711805343628,
+      "learning_rate": 2.677583025830259e-06,
+      "loss": 1.4442,
+      "step": 1264
+    },
+    {
+      "epoch": 1.5743621655258244,
+      "grad_norm": 2.8596792221069336,
+      "learning_rate": 2.675276752767528e-06,
+      "loss": 1.4325,
+      "step": 1265
+    },
+    {
+      "epoch": 1.5756067205973865,
+      "grad_norm": 2.9592769145965576,
+      "learning_rate": 2.6729704797047973e-06,
+      "loss": 1.4059,
+      "step": 1266
+    },
+    {
+      "epoch": 1.5768512756689483,
+      "grad_norm": 2.773589611053467,
+      "learning_rate": 2.6706642066420667e-06,
+      "loss": 1.3608,
+      "step": 1267
+    },
+    {
+      "epoch": 1.5780958307405104,
+      "grad_norm": 3.313255548477173,
+      "learning_rate": 2.6683579335793357e-06,
+      "loss": 1.4552,
+      "step": 1268
+    },
+    {
+      "epoch": 1.5793403858120723,
+      "grad_norm": 2.977649211883545,
+      "learning_rate": 2.6660516605166056e-06,
+      "loss": 1.4221,
+      "step": 1269
+    },
+    {
+      "epoch": 1.580584940883634,
+      "grad_norm": 3.0672051906585693,
+      "learning_rate": 2.663745387453875e-06,
+      "loss": 1.4568,
+      "step": 1270
+    },
+    {
+      "epoch": 1.580584940883634,
+      "eval_loss": 1.6317907571792603,
+      "eval_runtime": 47.2354,
+      "eval_samples_per_second": 21.171,
+      "eval_steps_per_second": 0.889,
+      "step": 1270
+    },
+    {
+      "epoch": 1.581829495955196,
+      "grad_norm": 3.029320240020752,
+      "learning_rate": 2.661439114391144e-06,
+      "loss": 1.4432,
+      "step": 1271
+    },
+    {
+      "epoch": 1.5830740510267578,
+      "grad_norm": 3.0183846950531006,
+      "learning_rate": 2.6591328413284134e-06,
+      "loss": 1.4332,
+      "step": 1272
+    },
+    {
+      "epoch": 1.5843186060983199,
+      "grad_norm": 2.856551170349121,
+      "learning_rate": 2.656826568265683e-06,
+      "loss": 1.4291,
+      "step": 1273
+    },
+    {
+      "epoch": 1.5855631611698817,
+      "grad_norm": 3.124971389770508,
+      "learning_rate": 2.6545202952029527e-06,
+      "loss": 1.4363,
+      "step": 1274
+    },
+    {
+      "epoch": 1.5868077162414438,
+      "grad_norm": 2.9470791816711426,
+      "learning_rate": 2.6522140221402217e-06,
+      "loss": 1.4502,
+      "step": 1275
+    },
+    {
+      "epoch": 1.5880522713130056,
+      "grad_norm": 2.9116358757019043,
+      "learning_rate": 2.649907749077491e-06,
+      "loss": 1.4314,
+      "step": 1276
+    },
+    {
+      "epoch": 1.5892968263845675,
+      "grad_norm": 3.2575323581695557,
+      "learning_rate": 2.6476014760147605e-06,
+      "loss": 1.4749,
+      "step": 1277
+    },
+    {
+      "epoch": 1.5905413814561293,
+      "grad_norm": 2.935518503189087,
+      "learning_rate": 2.6452952029520295e-06,
+      "loss": 1.4265,
+      "step": 1278
+    },
+    {
+      "epoch": 1.5917859365276914,
+      "grad_norm": 2.9750072956085205,
+      "learning_rate": 2.642988929889299e-06,
+      "loss": 1.4501,
+      "step": 1279
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "grad_norm": 2.8086845874786377,
+      "learning_rate": 2.640682656826569e-06,
+      "loss": 1.4557,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "eval_loss": 1.6302106380462646,
+      "eval_runtime": 52.3962,
+      "eval_samples_per_second": 19.085,
+      "eval_steps_per_second": 0.802,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5942750466708153,
+      "grad_norm": 2.741849422454834,
+      "learning_rate": 2.638376383763838e-06,
+      "loss": 1.3918,
+      "step": 1281
+    },
+    {
+      "epoch": 1.5955196017423772,
+      "grad_norm": 2.9414806365966797,
+      "learning_rate": 2.6360701107011072e-06,
+      "loss": 1.3882,
+      "step": 1282
+    },
+    {
+      "epoch": 1.596764156813939,
+      "grad_norm": 2.901514768600464,
+      "learning_rate": 2.6337638376383766e-06,
+      "loss": 1.4331,
+      "step": 1283
+    },
+    {
+      "epoch": 1.5980087118855009,
+      "grad_norm": 2.8159122467041016,
+      "learning_rate": 2.6314575645756456e-06,
+      "loss": 1.4064,
+      "step": 1284
+    },
+    {
+      "epoch": 1.5992532669570627,
+      "grad_norm": 2.900561571121216,
+      "learning_rate": 2.629151291512915e-06,
+      "loss": 1.3731,
+      "step": 1285
+    },
+    {
+      "epoch": 1.6004978220286248,
+      "grad_norm": 3.019540548324585,
+      "learning_rate": 2.626845018450185e-06,
+      "loss": 1.4506,
+      "step": 1286
+    },
+    {
+      "epoch": 1.6017423771001866,
+      "grad_norm": 2.9263620376586914,
+      "learning_rate": 2.6245387453874543e-06,
+      "loss": 1.4754,
+      "step": 1287
+    },
+    {
+      "epoch": 1.6029869321717487,
+      "grad_norm": 2.8681881427764893,
+      "learning_rate": 2.6222324723247233e-06,
+      "loss": 1.4124,
+      "step": 1288
+    },
+    {
+      "epoch": 1.6042314872433105,
+      "grad_norm": 2.922945261001587,
+      "learning_rate": 2.6199261992619928e-06,
+      "loss": 1.5101,
+      "step": 1289
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "grad_norm": 2.9548606872558594,
+      "learning_rate": 2.6176199261992626e-06,
+      "loss": 1.41,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "eval_loss": 1.6357502937316895,
+      "eval_runtime": 46.9547,
+      "eval_samples_per_second": 21.297,
+      "eval_steps_per_second": 0.894,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6067205973864342,
+      "grad_norm": 2.837184190750122,
+      "learning_rate": 2.6153136531365316e-06,
+      "loss": 1.4354,
+      "step": 1291
+    },
+    {
+      "epoch": 1.6079651524579963,
+      "grad_norm": 2.818990468978882,
+      "learning_rate": 2.613007380073801e-06,
+      "loss": 1.4474,
+      "step": 1292
+    },
+    {
+      "epoch": 1.6092097075295582,
+      "grad_norm": 2.7880146503448486,
+      "learning_rate": 2.6107011070110704e-06,
+      "loss": 1.4156,
+      "step": 1293
+    },
+    {
+      "epoch": 1.6104542626011202,
+      "grad_norm": 2.790971040725708,
+      "learning_rate": 2.6083948339483394e-06,
+      "loss": 1.4671,
+      "step": 1294
+    },
+    {
+      "epoch": 1.611698817672682,
+      "grad_norm": 2.943129777908325,
+      "learning_rate": 2.606088560885609e-06,
+      "loss": 1.4801,
+      "step": 1295
+    },
+    {
+      "epoch": 1.612943372744244,
+      "grad_norm": 2.8589110374450684,
+      "learning_rate": 2.6037822878228787e-06,
+      "loss": 1.4894,
+      "step": 1296
+    },
+    {
+      "epoch": 1.6141879278158058,
+      "grad_norm": 2.8929262161254883,
+      "learning_rate": 2.6014760147601477e-06,
+      "loss": 1.4022,
+      "step": 1297
+    },
+    {
+      "epoch": 1.6154324828873676,
+      "grad_norm": 2.970099925994873,
+      "learning_rate": 2.599169741697417e-06,
+      "loss": 1.5136,
+      "step": 1298
+    },
+    {
+      "epoch": 1.6166770379589297,
+      "grad_norm": 2.928466796875,
+      "learning_rate": 2.5968634686346866e-06,
+      "loss": 1.4743,
+      "step": 1299
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "grad_norm": 2.941580057144165,
+      "learning_rate": 2.594557195571956e-06,
+      "loss": 1.4397,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "eval_loss": 1.6348339319229126,
+      "eval_runtime": 50.433,
+      "eval_samples_per_second": 19.828,
+      "eval_steps_per_second": 0.833,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6191661481020536,
+      "grad_norm": 2.903212308883667,
+      "learning_rate": 2.592250922509225e-06,
+      "loss": 1.4221,
+      "step": 1301
+    },
+    {
+      "epoch": 1.6204107031736155,
+      "grad_norm": 2.948651075363159,
+      "learning_rate": 2.589944649446495e-06,
+      "loss": 1.4363,
+      "step": 1302
+    },
+    {
+      "epoch": 1.6216552582451773,
+      "grad_norm": 2.8940505981445312,
+      "learning_rate": 2.5876383763837643e-06,
+      "loss": 1.398,
+      "step": 1303
+    },
+    {
+      "epoch": 1.6228998133167392,
+      "grad_norm": 2.950056552886963,
+      "learning_rate": 2.5853321033210333e-06,
+      "loss": 1.4425,
+      "step": 1304
+    },
+    {
+      "epoch": 1.6241443683883012,
+      "grad_norm": 2.9674088954925537,
+      "learning_rate": 2.5830258302583027e-06,
+      "loss": 1.4473,
+      "step": 1305
+    },
+    {
+      "epoch": 1.625388923459863,
+      "grad_norm": 2.7541685104370117,
+      "learning_rate": 2.5807195571955725e-06,
+      "loss": 1.4483,
+      "step": 1306
+    },
+    {
+      "epoch": 1.6266334785314251,
+      "grad_norm": 2.8897035121917725,
+      "learning_rate": 2.5784132841328415e-06,
+      "loss": 1.4572,
+      "step": 1307
+    },
+    {
+      "epoch": 1.627878033602987,
+      "grad_norm": 3.021559238433838,
+      "learning_rate": 2.576107011070111e-06,
+      "loss": 1.5164,
+      "step": 1308
+    },
+    {
+      "epoch": 1.6291225886745488,
+      "grad_norm": 2.884009838104248,
+      "learning_rate": 2.5738007380073804e-06,
+      "loss": 1.4196,
+      "step": 1309
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "grad_norm": 2.749497890472412,
+      "learning_rate": 2.5714944649446494e-06,
+      "loss": 1.4324,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "eval_loss": 1.6314265727996826,
+      "eval_runtime": 42.4647,
+      "eval_samples_per_second": 23.549,
+      "eval_steps_per_second": 0.989,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6316116988176725,
+      "grad_norm": 2.8908045291900635,
+      "learning_rate": 2.5691881918819188e-06,
+      "loss": 1.4388,
+      "step": 1311
+    },
+    {
+      "epoch": 1.6328562538892346,
+      "grad_norm": 2.832808256149292,
+      "learning_rate": 2.5668819188191886e-06,
+      "loss": 1.4368,
+      "step": 1312
+    },
+    {
+      "epoch": 1.6341008089607967,
+      "grad_norm": 2.9047906398773193,
+      "learning_rate": 2.564575645756458e-06,
+      "loss": 1.4186,
+      "step": 1313
+    },
+    {
+      "epoch": 1.6353453640323585,
+      "grad_norm": 2.9271419048309326,
+      "learning_rate": 2.562269372693727e-06,
+      "loss": 1.3776,
+      "step": 1314
+    },
+    {
+      "epoch": 1.6365899191039204,
+      "grad_norm": 2.760314702987671,
+      "learning_rate": 2.5599630996309965e-06,
+      "loss": 1.386,
+      "step": 1315
+    },
+    {
+      "epoch": 1.6378344741754822,
+      "grad_norm": 2.9694864749908447,
+      "learning_rate": 2.557656826568266e-06,
+      "loss": 1.4542,
+      "step": 1316
+    },
+    {
+      "epoch": 1.639079029247044,
+      "grad_norm": 2.9950435161590576,
+      "learning_rate": 2.555350553505535e-06,
+      "loss": 1.4242,
+      "step": 1317
+    },
+    {
+      "epoch": 1.6403235843186061,
+      "grad_norm": 2.9291114807128906,
+      "learning_rate": 2.5530442804428047e-06,
+      "loss": 1.4289,
+      "step": 1318
+    },
+    {
+      "epoch": 1.641568139390168,
+      "grad_norm": 2.981961488723755,
+      "learning_rate": 2.550738007380074e-06,
+      "loss": 1.4283,
+      "step": 1319
+    },
+    {
+      "epoch": 1.64281269446173,
+      "grad_norm": 2.734123706817627,
+      "learning_rate": 2.548431734317343e-06,
+      "loss": 1.4144,
+      "step": 1320
+    },
+    {
+      "epoch": 1.64281269446173,
+      "eval_loss": 1.6249239444732666,
+      "eval_runtime": 48.8116,
+      "eval_samples_per_second": 20.487,
+      "eval_steps_per_second": 0.86,
+      "step": 1320
+    },
+    {
+      "epoch": 1.644057249533292,
+      "grad_norm": 2.9752919673919678,
+      "learning_rate": 2.5461254612546126e-06,
+      "loss": 1.3641,
+      "step": 1321
+    },
+    {
+      "epoch": 1.6453018046048538,
+      "grad_norm": 2.972463607788086,
+      "learning_rate": 2.543819188191882e-06,
+      "loss": 1.3797,
+      "step": 1322
+    },
+    {
+      "epoch": 1.6465463596764156,
+      "grad_norm": 2.905499219894409,
+      "learning_rate": 2.541512915129152e-06,
+      "loss": 1.4221,
+      "step": 1323
+    },
+    {
+      "epoch": 1.6477909147479775,
+      "grad_norm": 2.7921793460845947,
+      "learning_rate": 2.539206642066421e-06,
+      "loss": 1.4118,
+      "step": 1324
+    },
+    {
+      "epoch": 1.6490354698195395,
+      "grad_norm": 3.064558267593384,
+      "learning_rate": 2.5369003690036903e-06,
+      "loss": 1.4611,
+      "step": 1325
+    },
+    {
+      "epoch": 1.6502800248911016,
+      "grad_norm": 2.9690096378326416,
+      "learning_rate": 2.5345940959409597e-06,
+      "loss": 1.461,
+      "step": 1326
+    },
+    {
+      "epoch": 1.6515245799626634,
+      "grad_norm": 3.0379040241241455,
+      "learning_rate": 2.5322878228782287e-06,
+      "loss": 1.451,
+      "step": 1327
+    },
+    {
+      "epoch": 1.6527691350342253,
+      "grad_norm": 2.9426493644714355,
+      "learning_rate": 2.5299815498154986e-06,
+      "loss": 1.4671,
+      "step": 1328
+    },
+    {
+      "epoch": 1.6540136901057871,
+      "grad_norm": 2.915076732635498,
+      "learning_rate": 2.527675276752768e-06,
+      "loss": 1.4159,
+      "step": 1329
+    },
+    {
+      "epoch": 1.655258245177349,
+      "grad_norm": 2.8588879108428955,
+      "learning_rate": 2.525369003690037e-06,
+      "loss": 1.4978,
+      "step": 1330
+    },
+    {
+      "epoch": 1.655258245177349,
+      "eval_loss": 1.624009132385254,
+      "eval_runtime": 47.0461,
+      "eval_samples_per_second": 21.256,
+      "eval_steps_per_second": 0.893,
+      "step": 1330
+    },
+    {
+      "epoch": 1.656502800248911,
+      "grad_norm": 2.9942209720611572,
+      "learning_rate": 2.5230627306273064e-06,
+      "loss": 1.3872,
+      "step": 1331
+    },
+    {
+      "epoch": 1.657747355320473,
+      "grad_norm": 2.8424410820007324,
+      "learning_rate": 2.520756457564576e-06,
+      "loss": 1.4181,
+      "step": 1332
+    },
+    {
+      "epoch": 1.658991910392035,
+      "grad_norm": 2.9234843254089355,
+      "learning_rate": 2.518450184501845e-06,
+      "loss": 1.4516,
+      "step": 1333
+    },
+    {
+      "epoch": 1.6602364654635968,
+      "grad_norm": 2.8367815017700195,
+      "learning_rate": 2.5161439114391147e-06,
+      "loss": 1.4214,
+      "step": 1334
+    },
+    {
+      "epoch": 1.6614810205351587,
+      "grad_norm": 2.819149971008301,
+      "learning_rate": 2.513837638376384e-06,
+      "loss": 1.3883,
+      "step": 1335
+    },
+    {
+      "epoch": 1.6627255756067205,
+      "grad_norm": 3.024693250656128,
+      "learning_rate": 2.5115313653136535e-06,
+      "loss": 1.3902,
+      "step": 1336
+    },
+    {
+      "epoch": 1.6639701306782824,
+      "grad_norm": 2.903599262237549,
+      "learning_rate": 2.5092250922509225e-06,
+      "loss": 1.4238,
+      "step": 1337
+    },
+    {
+      "epoch": 1.6652146857498444,
+      "grad_norm": 2.8296589851379395,
+      "learning_rate": 2.506918819188192e-06,
+      "loss": 1.3993,
+      "step": 1338
+    },
+    {
+      "epoch": 1.6664592408214065,
+      "grad_norm": 2.873272657394409,
+      "learning_rate": 2.5046125461254618e-06,
+      "loss": 1.4186,
+      "step": 1339
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "grad_norm": 2.9268789291381836,
+      "learning_rate": 2.5023062730627308e-06,
+      "loss": 1.4242,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "eval_loss": 1.6260672807693481,
+      "eval_runtime": 47.5895,
+      "eval_samples_per_second": 21.013,
+      "eval_steps_per_second": 0.883,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6689483509645302,
+      "grad_norm": 3.057659387588501,
+      "learning_rate": 2.5e-06,
+      "loss": 1.4077,
+      "step": 1341
+    },
+    {
+      "epoch": 1.670192906036092,
+      "grad_norm": 2.875095844268799,
+      "learning_rate": 2.4976937269372696e-06,
+      "loss": 1.394,
+      "step": 1342
+    },
+    {
+      "epoch": 1.671437461107654,
+      "grad_norm": 2.7579872608184814,
+      "learning_rate": 2.495387453874539e-06,
+      "loss": 1.4029,
+      "step": 1343
+    },
+    {
+      "epoch": 1.672682016179216,
+      "grad_norm": 2.8981001377105713,
+      "learning_rate": 2.4930811808118085e-06,
+      "loss": 1.4494,
+      "step": 1344
+    },
+    {
+      "epoch": 1.6739265712507778,
+      "grad_norm": 2.9696900844573975,
+      "learning_rate": 2.490774907749078e-06,
+      "loss": 1.4692,
+      "step": 1345
+    },
+    {
+      "epoch": 1.6751711263223399,
+      "grad_norm": 2.8094892501831055,
+      "learning_rate": 2.488468634686347e-06,
+      "loss": 1.441,
+      "step": 1346
+    },
+    {
+      "epoch": 1.6764156813939017,
+      "grad_norm": 2.8619563579559326,
+      "learning_rate": 2.4861623616236163e-06,
+      "loss": 1.4885,
+      "step": 1347
+    },
+    {
+      "epoch": 1.6776602364654636,
+      "grad_norm": 2.854151487350464,
+      "learning_rate": 2.4838560885608857e-06,
+      "loss": 1.4338,
+      "step": 1348
+    },
+    {
+      "epoch": 1.6789047915370254,
+      "grad_norm": 2.9707868099212646,
+      "learning_rate": 2.481549815498155e-06,
+      "loss": 1.4529,
+      "step": 1349
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "grad_norm": 3.0289969444274902,
+      "learning_rate": 2.4792435424354246e-06,
+      "loss": 1.4855,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "eval_loss": 1.6256393194198608,
+      "eval_runtime": 45.9524,
+      "eval_samples_per_second": 21.762,
+      "eval_steps_per_second": 0.914,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6813939016801493,
+      "grad_norm": 2.8407845497131348,
+      "learning_rate": 2.476937269372694e-06,
+      "loss": 1.3876,
+      "step": 1351
+    },
+    {
+      "epoch": 1.6826384567517114,
+      "grad_norm": 2.9876599311828613,
+      "learning_rate": 2.4746309963099634e-06,
+      "loss": 1.3931,
+      "step": 1352
+    },
+    {
+      "epoch": 1.6838830118232733,
+      "grad_norm": 3.008683919906616,
+      "learning_rate": 2.472324723247233e-06,
+      "loss": 1.4482,
+      "step": 1353
+    },
+    {
+      "epoch": 1.6851275668948351,
+      "grad_norm": 2.8110079765319824,
+      "learning_rate": 2.470018450184502e-06,
+      "loss": 1.3851,
+      "step": 1354
+    },
+    {
+      "epoch": 1.686372121966397,
+      "grad_norm": 2.9402263164520264,
+      "learning_rate": 2.4677121771217713e-06,
+      "loss": 1.4009,
+      "step": 1355
+    },
+    {
+      "epoch": 1.6876166770379588,
+      "grad_norm": 2.9846489429473877,
+      "learning_rate": 2.4654059040590407e-06,
+      "loss": 1.3975,
+      "step": 1356
+    },
+    {
+      "epoch": 1.6888612321095209,
+      "grad_norm": 2.835801362991333,
+      "learning_rate": 2.46309963099631e-06,
+      "loss": 1.3421,
+      "step": 1357
+    },
+    {
+      "epoch": 1.6901057871810827,
+      "grad_norm": 2.9438889026641846,
+      "learning_rate": 2.4607933579335795e-06,
+      "loss": 1.4422,
+      "step": 1358
+    },
+    {
+      "epoch": 1.6913503422526448,
+      "grad_norm": 3.168829917907715,
+      "learning_rate": 2.458487084870849e-06,
+      "loss": 1.4596,
+      "step": 1359
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "grad_norm": 2.8412039279937744,
+      "learning_rate": 2.4561808118081184e-06,
+      "loss": 1.4263,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "eval_loss": 1.6265885829925537,
+      "eval_runtime": 50.3402,
+      "eval_samples_per_second": 19.865,
+      "eval_steps_per_second": 0.834,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6938394523957685,
+      "grad_norm": 3.0540060997009277,
+      "learning_rate": 2.453874538745388e-06,
+      "loss": 1.4123,
+      "step": 1361
+    },
+    {
+      "epoch": 1.6950840074673303,
+      "grad_norm": 3.0544989109039307,
+      "learning_rate": 2.451568265682657e-06,
+      "loss": 1.412,
+      "step": 1362
+    },
+    {
+      "epoch": 1.6963285625388922,
+      "grad_norm": 3.114126205444336,
+      "learning_rate": 2.4492619926199267e-06,
+      "loss": 1.4952,
+      "step": 1363
+    },
+    {
+      "epoch": 1.6975731176104543,
+      "grad_norm": 2.9366016387939453,
+      "learning_rate": 2.4469557195571957e-06,
+      "loss": 1.5155,
+      "step": 1364
+    },
+    {
+      "epoch": 1.6988176726820163,
+      "grad_norm": 2.910677671432495,
+      "learning_rate": 2.444649446494465e-06,
+      "loss": 1.4393,
+      "step": 1365
+    },
+    {
+      "epoch": 1.7000622277535782,
+      "grad_norm": 2.8373727798461914,
+      "learning_rate": 2.4423431734317345e-06,
+      "loss": 1.4223,
+      "step": 1366
+    },
+    {
+      "epoch": 1.70130678282514,
+      "grad_norm": 2.865161895751953,
+      "learning_rate": 2.440036900369004e-06,
+      "loss": 1.4605,
+      "step": 1367
+    },
+    {
+      "epoch": 1.7025513378967019,
+      "grad_norm": 2.7929177284240723,
+      "learning_rate": 2.437730627306273e-06,
+      "loss": 1.4695,
+      "step": 1368
+    },
+    {
+      "epoch": 1.7037958929682637,
+      "grad_norm": 2.867953300476074,
+      "learning_rate": 2.4354243542435428e-06,
+      "loss": 1.3524,
+      "step": 1369
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "grad_norm": 2.8980207443237305,
+      "learning_rate": 2.4331180811808118e-06,
+      "loss": 1.4568,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "eval_loss": 1.6297556161880493,
+      "eval_runtime": 51.447,
+      "eval_samples_per_second": 19.437,
+      "eval_steps_per_second": 0.816,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7062850031113876,
+      "grad_norm": 2.954972505569458,
+      "learning_rate": 2.4308118081180816e-06,
+      "loss": 1.4,
+      "step": 1371
+    },
+    {
+      "epoch": 1.7075295581829497,
+      "grad_norm": 2.973191738128662,
+      "learning_rate": 2.4285055350553506e-06,
+      "loss": 1.4062,
+      "step": 1372
+    },
+    {
+      "epoch": 1.7087741132545116,
+      "grad_norm": 2.951444387435913,
+      "learning_rate": 2.42619926199262e-06,
+      "loss": 1.4523,
+      "step": 1373
+    },
+    {
+      "epoch": 1.7100186683260734,
+      "grad_norm": 2.850445508956909,
+      "learning_rate": 2.4238929889298895e-06,
+      "loss": 1.4282,
+      "step": 1374
+    },
+    {
+      "epoch": 1.7112632233976353,
+      "grad_norm": 2.7643561363220215,
+      "learning_rate": 2.421586715867159e-06,
+      "loss": 1.3807,
+      "step": 1375
+    },
+    {
+      "epoch": 1.712507778469197,
+      "grad_norm": 2.8872134685516357,
+      "learning_rate": 2.4192804428044283e-06,
+      "loss": 1.4801,
+      "step": 1376
+    },
+    {
+      "epoch": 1.7137523335407592,
+      "grad_norm": 2.8748483657836914,
+      "learning_rate": 2.4169741697416977e-06,
+      "loss": 1.381,
+      "step": 1377
+    },
+    {
+      "epoch": 1.7149968886123212,
+      "grad_norm": 2.7833831310272217,
+      "learning_rate": 2.4146678966789667e-06,
+      "loss": 1.389,
+      "step": 1378
+    },
+    {
+      "epoch": 1.716241443683883,
+      "grad_norm": 2.994715690612793,
+      "learning_rate": 2.4123616236162366e-06,
+      "loss": 1.3917,
+      "step": 1379
+    },
+    {
+      "epoch": 1.717485998755445,
+      "grad_norm": 2.887026786804199,
+      "learning_rate": 2.4100553505535056e-06,
+      "loss": 1.3725,
+      "step": 1380
+    },
+    {
+      "epoch": 1.717485998755445,
+      "eval_loss": 1.6283434629440308,
+      "eval_runtime": 50.596,
+      "eval_samples_per_second": 19.764,
+      "eval_steps_per_second": 0.83,
+      "step": 1380
+    },
+    {
+      "epoch": 1.7187305538270068,
+      "grad_norm": 3.2046546936035156,
+      "learning_rate": 2.407749077490775e-06,
+      "loss": 1.4422,
+      "step": 1381
+    },
+    {
+      "epoch": 1.7199751088985686,
+      "grad_norm": 3.0105698108673096,
+      "learning_rate": 2.4054428044280444e-06,
+      "loss": 1.4494,
+      "step": 1382
+    },
+    {
+      "epoch": 1.7212196639701307,
+      "grad_norm": 2.9917800426483154,
+      "learning_rate": 2.403136531365314e-06,
+      "loss": 1.4407,
+      "step": 1383
+    },
+    {
+      "epoch": 1.7224642190416926,
+      "grad_norm": 2.844325065612793,
+      "learning_rate": 2.4008302583025833e-06,
+      "loss": 1.4067,
+      "step": 1384
+    },
+    {
+      "epoch": 1.7237087741132546,
+      "grad_norm": 2.919576406478882,
+      "learning_rate": 2.3985239852398527e-06,
+      "loss": 1.4352,
+      "step": 1385
+    },
+    {
+      "epoch": 1.7249533291848165,
+      "grad_norm": 2.8357744216918945,
+      "learning_rate": 2.3962177121771217e-06,
+      "loss": 1.3978,
+      "step": 1386
+    },
+    {
+      "epoch": 1.7261978842563783,
+      "grad_norm": 2.82425856590271,
+      "learning_rate": 2.3939114391143915e-06,
+      "loss": 1.4439,
+      "step": 1387
+    },
+    {
+      "epoch": 1.7274424393279402,
+      "grad_norm": 2.9155638217926025,
+      "learning_rate": 2.3916051660516605e-06,
+      "loss": 1.4326,
+      "step": 1388
+    },
+    {
+      "epoch": 1.728686994399502,
+      "grad_norm": 2.862804651260376,
+      "learning_rate": 2.3892988929889304e-06,
+      "loss": 1.3766,
+      "step": 1389
+    },
+    {
+      "epoch": 1.729931549471064,
+      "grad_norm": 3.012230634689331,
+      "learning_rate": 2.3869926199261994e-06,
+      "loss": 1.4569,
+      "step": 1390
+    },
+    {
+      "epoch": 1.729931549471064,
+      "eval_loss": 1.6268510818481445,
+      "eval_runtime": 52.5438,
+      "eval_samples_per_second": 19.032,
+      "eval_steps_per_second": 0.799,
+      "step": 1390
+    },
+    {
+      "epoch": 1.7311761045426262,
+      "grad_norm": 2.9922425746917725,
+      "learning_rate": 2.384686346863469e-06,
+      "loss": 1.4066,
+      "step": 1391
+    },
+    {
+      "epoch": 1.732420659614188,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 2.3823800738007382e-06,
+      "loss": 1.4514,
+      "step": 1392
+    },
+    {
+      "epoch": 1.7336652146857499,
+      "grad_norm": 2.834998846054077,
+      "learning_rate": 2.3800738007380077e-06,
+      "loss": 1.4714,
+      "step": 1393
+    },
+    {
+      "epoch": 1.7349097697573117,
+      "grad_norm": 2.9646077156066895,
+      "learning_rate": 2.377767527675277e-06,
+      "loss": 1.4368,
+      "step": 1394
+    },
+    {
+      "epoch": 1.7361543248288736,
+      "grad_norm": 2.8233699798583984,
+      "learning_rate": 2.3754612546125465e-06,
+      "loss": 1.4598,
+      "step": 1395
+    },
+    {
+      "epoch": 1.7373988799004356,
+      "grad_norm": 2.895472764968872,
+      "learning_rate": 2.3731549815498155e-06,
+      "loss": 1.4551,
+      "step": 1396
+    },
+    {
+      "epoch": 1.7386434349719975,
+      "grad_norm": 2.7917752265930176,
+      "learning_rate": 2.3708487084870853e-06,
+      "loss": 1.4759,
+      "step": 1397
+    },
+    {
+      "epoch": 1.7398879900435595,
+      "grad_norm": 2.820672035217285,
+      "learning_rate": 2.3685424354243543e-06,
+      "loss": 1.3711,
+      "step": 1398
+    },
+    {
+      "epoch": 1.7411325451151214,
+      "grad_norm": 2.804952621459961,
+      "learning_rate": 2.3662361623616238e-06,
+      "loss": 1.4227,
+      "step": 1399
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "grad_norm": 2.981553316116333,
+      "learning_rate": 2.363929889298893e-06,
+      "loss": 1.4639,
+      "step": 1400
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "eval_loss": 1.6229957342147827,
+      "eval_runtime": 49.1507,
+      "eval_samples_per_second": 20.346,
+      "eval_steps_per_second": 0.855,
+      "step": 1400
+    },
+    {
+      "epoch": 1.743621655258245,
+      "grad_norm": 2.8649392127990723,
+      "learning_rate": 2.3616236162361626e-06,
+      "loss": 1.4283,
+      "step": 1401
+    },
+    {
+      "epoch": 1.744866210329807,
+      "grad_norm": 2.9438977241516113,
+      "learning_rate": 2.359317343173432e-06,
+      "loss": 1.444,
+      "step": 1402
+    },
+    {
+      "epoch": 1.746110765401369,
+      "grad_norm": 2.9270176887512207,
+      "learning_rate": 2.3570110701107015e-06,
+      "loss": 1.4044,
+      "step": 1403
+    },
+    {
+      "epoch": 1.747355320472931,
+      "grad_norm": 2.791527271270752,
+      "learning_rate": 2.3547047970479705e-06,
+      "loss": 1.4195,
+      "step": 1404
+    },
+    {
+      "epoch": 1.748599875544493,
+      "grad_norm": 2.8519608974456787,
+      "learning_rate": 2.3523985239852403e-06,
+      "loss": 1.4031,
+      "step": 1405
+    },
+    {
+      "epoch": 1.7498444306160548,
+      "grad_norm": 2.90872859954834,
+      "learning_rate": 2.3500922509225093e-06,
+      "loss": 1.4616,
+      "step": 1406
+    },
+    {
+      "epoch": 1.7510889856876166,
+      "grad_norm": 2.9190874099731445,
+      "learning_rate": 2.3477859778597787e-06,
+      "loss": 1.3917,
+      "step": 1407
+    },
+    {
+      "epoch": 1.7523335407591785,
+      "grad_norm": 2.868323802947998,
+      "learning_rate": 2.345479704797048e-06,
+      "loss": 1.4053,
+      "step": 1408
+    },
+    {
+      "epoch": 1.7535780958307405,
+      "grad_norm": 2.8638579845428467,
+      "learning_rate": 2.3431734317343176e-06,
+      "loss": 1.4042,
+      "step": 1409
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "grad_norm": 3.1759817600250244,
+      "learning_rate": 2.340867158671587e-06,
+      "loss": 1.4937,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "eval_loss": 1.6252617835998535,
+      "eval_runtime": 44.3041,
+      "eval_samples_per_second": 22.571,
+      "eval_steps_per_second": 0.948,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7560672059738645,
+      "grad_norm": 2.8265459537506104,
+      "learning_rate": 2.3385608856088564e-06,
+      "loss": 1.406,
+      "step": 1411
+    },
+    {
+      "epoch": 1.7573117610454263,
+      "grad_norm": 2.856229305267334,
+      "learning_rate": 2.336254612546126e-06,
+      "loss": 1.4395,
+      "step": 1412
+    },
+    {
+      "epoch": 1.7585563161169881,
+      "grad_norm": 2.9112775325775146,
+      "learning_rate": 2.333948339483395e-06,
+      "loss": 1.427,
+      "step": 1413
+    },
+    {
+      "epoch": 1.75980087118855,
+      "grad_norm": 3.098846435546875,
+      "learning_rate": 2.3316420664206643e-06,
+      "loss": 1.4222,
+      "step": 1414
+    },
+    {
+      "epoch": 1.7610454262601118,
+      "grad_norm": 2.758100986480713,
+      "learning_rate": 2.3293357933579337e-06,
+      "loss": 1.3844,
+      "step": 1415
+    },
+    {
+      "epoch": 1.762289981331674,
+      "grad_norm": 2.8422274589538574,
+      "learning_rate": 2.327029520295203e-06,
+      "loss": 1.4033,
+      "step": 1416
+    },
+    {
+      "epoch": 1.763534536403236,
+      "grad_norm": 2.9257912635803223,
+      "learning_rate": 2.3247232472324725e-06,
+      "loss": 1.3961,
+      "step": 1417
+    },
+    {
+      "epoch": 1.7647790914747978,
+      "grad_norm": 2.9431657791137695,
+      "learning_rate": 2.322416974169742e-06,
+      "loss": 1.4451,
+      "step": 1418
+    },
+    {
+      "epoch": 1.7660236465463597,
+      "grad_norm": 2.8598814010620117,
+      "learning_rate": 2.3201107011070114e-06,
+      "loss": 1.3816,
+      "step": 1419
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "grad_norm": 2.9868006706237793,
+      "learning_rate": 2.317804428044281e-06,
+      "loss": 1.4037,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "eval_loss": 1.6231273412704468,
+      "eval_runtime": 48.8067,
+      "eval_samples_per_second": 20.489,
+      "eval_steps_per_second": 0.861,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7685127566894834,
+      "grad_norm": 2.888206720352173,
+      "learning_rate": 2.31549815498155e-06,
+      "loss": 1.4087,
+      "step": 1421
+    },
+    {
+      "epoch": 1.7697573117610454,
+      "grad_norm": 2.9240078926086426,
+      "learning_rate": 2.3131918819188192e-06,
+      "loss": 1.3909,
+      "step": 1422
+    },
+    {
+      "epoch": 1.7710018668326073,
+      "grad_norm": 3.0401012897491455,
+      "learning_rate": 2.3108856088560886e-06,
+      "loss": 1.423,
+      "step": 1423
+    },
+    {
+      "epoch": 1.7722464219041694,
+      "grad_norm": 2.9484200477600098,
+      "learning_rate": 2.308579335793358e-06,
+      "loss": 1.449,
+      "step": 1424
+    },
+    {
+      "epoch": 1.7734909769757312,
+      "grad_norm": 2.863267660140991,
+      "learning_rate": 2.3062730627306275e-06,
+      "loss": 1.4215,
+      "step": 1425
+    },
+    {
+      "epoch": 1.774735532047293,
+      "grad_norm": 2.9615612030029297,
+      "learning_rate": 2.303966789667897e-06,
+      "loss": 1.4391,
+      "step": 1426
+    },
+    {
+      "epoch": 1.775980087118855,
+      "grad_norm": 2.7958569526672363,
+      "learning_rate": 2.3016605166051663e-06,
+      "loss": 1.3662,
+      "step": 1427
+    },
+    {
+      "epoch": 1.777224642190417,
+      "grad_norm": 2.898787021636963,
+      "learning_rate": 2.2993542435424358e-06,
+      "loss": 1.362,
+      "step": 1428
+    },
+    {
+      "epoch": 1.7784691972619788,
+      "grad_norm": 2.883049726486206,
+      "learning_rate": 2.2970479704797048e-06,
+      "loss": 1.4339,
+      "step": 1429
+    },
+    {
+      "epoch": 1.779713752333541,
+      "grad_norm": 3.0204827785491943,
+      "learning_rate": 2.294741697416974e-06,
+      "loss": 1.5174,
+      "step": 1430
+    },
+    {
+      "epoch": 1.779713752333541,
+      "eval_loss": 1.6246228218078613,
+      "eval_runtime": 47.5611,
+      "eval_samples_per_second": 21.026,
+      "eval_steps_per_second": 0.883,
+      "step": 1430
+    },
+    {
+      "epoch": 1.7809583074051027,
+      "grad_norm": 2.9354310035705566,
+      "learning_rate": 2.2924354243542436e-06,
+      "loss": 1.4115,
+      "step": 1431
+    },
+    {
+      "epoch": 1.7822028624766646,
+      "grad_norm": 2.9498608112335205,
+      "learning_rate": 2.290129151291513e-06,
+      "loss": 1.4058,
+      "step": 1432
+    },
+    {
+      "epoch": 1.7834474175482264,
+      "grad_norm": 2.9706921577453613,
+      "learning_rate": 2.2878228782287825e-06,
+      "loss": 1.426,
+      "step": 1433
+    },
+    {
+      "epoch": 1.7846919726197883,
+      "grad_norm": 2.8945024013519287,
+      "learning_rate": 2.285516605166052e-06,
+      "loss": 1.4011,
+      "step": 1434
+    },
+    {
+      "epoch": 1.7859365276913504,
+      "grad_norm": 2.8987081050872803,
+      "learning_rate": 2.2832103321033213e-06,
+      "loss": 1.4163,
+      "step": 1435
+    },
+    {
+      "epoch": 1.7871810827629122,
+      "grad_norm": 2.9797592163085938,
+      "learning_rate": 2.2809040590405907e-06,
+      "loss": 1.4367,
+      "step": 1436
+    },
+    {
+      "epoch": 1.7884256378344743,
+      "grad_norm": 2.8872063159942627,
+      "learning_rate": 2.2785977859778597e-06,
+      "loss": 1.4226,
+      "step": 1437
+    },
+    {
+      "epoch": 1.7896701929060361,
+      "grad_norm": 2.942655563354492,
+      "learning_rate": 2.2762915129151296e-06,
+      "loss": 1.4132,
+      "step": 1438
+    },
+    {
+      "epoch": 1.790914747977598,
+      "grad_norm": 3.027392625808716,
+      "learning_rate": 2.2739852398523986e-06,
+      "loss": 1.4148,
+      "step": 1439
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "grad_norm": 2.9475181102752686,
+      "learning_rate": 2.271678966789668e-06,
+      "loss": 1.4585,
+      "step": 1440
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "eval_loss": 1.6234897375106812,
+      "eval_runtime": 53.3737,
+      "eval_samples_per_second": 18.736,
+      "eval_steps_per_second": 0.787,
+      "step": 1440
+    },
+    {
+      "epoch": 1.793403858120722,
+      "grad_norm": 2.846304416656494,
+      "learning_rate": 2.2693726937269374e-06,
+      "loss": 1.3667,
+      "step": 1441
+    },
+    {
+      "epoch": 1.7946484131922837,
+      "grad_norm": 2.8925700187683105,
+      "learning_rate": 2.267066420664207e-06,
+      "loss": 1.4318,
+      "step": 1442
+    },
+    {
+      "epoch": 1.7958929682638458,
+      "grad_norm": 2.884025812149048,
+      "learning_rate": 2.2647601476014763e-06,
+      "loss": 1.4395,
+      "step": 1443
+    },
+    {
+      "epoch": 1.7971375233354077,
+      "grad_norm": 2.9763753414154053,
+      "learning_rate": 2.2624538745387457e-06,
+      "loss": 1.3742,
+      "step": 1444
+    },
+    {
+      "epoch": 1.7983820784069695,
+      "grad_norm": 3.0834898948669434,
+      "learning_rate": 2.2601476014760147e-06,
+      "loss": 1.4651,
+      "step": 1445
+    },
+    {
+      "epoch": 1.7996266334785314,
+      "grad_norm": 3.0673978328704834,
+      "learning_rate": 2.2578413284132845e-06,
+      "loss": 1.475,
+      "step": 1446
+    },
+    {
+      "epoch": 1.8008711885500932,
+      "grad_norm": 2.931521415710449,
+      "learning_rate": 2.2555350553505535e-06,
+      "loss": 1.4506,
+      "step": 1447
+    },
+    {
+      "epoch": 1.8021157436216553,
+      "grad_norm": 2.953301429748535,
+      "learning_rate": 2.253228782287823e-06,
+      "loss": 1.4224,
+      "step": 1448
+    },
+    {
+      "epoch": 1.8033602986932171,
+      "grad_norm": 3.0210883617401123,
+      "learning_rate": 2.2509225092250924e-06,
+      "loss": 1.4387,
+      "step": 1449
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "grad_norm": 2.975741386413574,
+      "learning_rate": 2.248616236162362e-06,
+      "loss": 1.4318,
+      "step": 1450
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "eval_loss": 1.618930697441101,
+      "eval_runtime": 49.6692,
+      "eval_samples_per_second": 20.133,
+      "eval_steps_per_second": 0.846,
+      "step": 1450
+    },
+    {
+      "epoch": 1.805849408836341,
+      "grad_norm": 2.9083876609802246,
+      "learning_rate": 2.2463099630996312e-06,
+      "loss": 1.3837,
+      "step": 1451
+    },
+    {
+      "epoch": 1.807093963907903,
+      "grad_norm": 3.364248275756836,
+      "learning_rate": 2.2440036900369006e-06,
+      "loss": 1.4589,
+      "step": 1452
+    },
+    {
+      "epoch": 1.8083385189794647,
+      "grad_norm": 2.935478925704956,
+      "learning_rate": 2.2416974169741696e-06,
+      "loss": 1.4119,
+      "step": 1453
+    },
+    {
+      "epoch": 1.8095830740510268,
+      "grad_norm": 3.1742911338806152,
+      "learning_rate": 2.2393911439114395e-06,
+      "loss": 1.4203,
+      "step": 1454
+    },
+    {
+      "epoch": 1.8108276291225887,
+      "grad_norm": 2.96966290473938,
+      "learning_rate": 2.2370848708487085e-06,
+      "loss": 1.4054,
+      "step": 1455
+    },
+    {
+      "epoch": 1.8120721841941507,
+      "grad_norm": 3.0919055938720703,
+      "learning_rate": 2.2347785977859783e-06,
+      "loss": 1.4289,
+      "step": 1456
+    },
+    {
+      "epoch": 1.8133167392657126,
+      "grad_norm": 2.9061055183410645,
+      "learning_rate": 2.2324723247232473e-06,
+      "loss": 1.4958,
+      "step": 1457
+    },
+    {
+      "epoch": 1.8145612943372744,
+      "grad_norm": 2.968348503112793,
+      "learning_rate": 2.2301660516605168e-06,
+      "loss": 1.4033,
+      "step": 1458
+    },
+    {
+      "epoch": 1.8158058494088363,
+      "grad_norm": 3.0435378551483154,
+      "learning_rate": 2.227859778597786e-06,
+      "loss": 1.414,
+      "step": 1459
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "grad_norm": 2.9067916870117188,
+      "learning_rate": 2.2255535055350556e-06,
+      "loss": 1.3954,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "eval_loss": 1.6240814924240112,
+      "eval_runtime": 51.7164,
+      "eval_samples_per_second": 19.336,
+      "eval_steps_per_second": 0.812,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8182949595519602,
+      "grad_norm": 3.156384229660034,
+      "learning_rate": 2.2232472324723246e-06,
+      "loss": 1.4658,
+      "step": 1461
+    },
+    {
+      "epoch": 1.819539514623522,
+      "grad_norm": 2.9885313510894775,
+      "learning_rate": 2.2209409594095944e-06,
+      "loss": 1.4123,
+      "step": 1462
+    },
+    {
+      "epoch": 1.820784069695084,
+      "grad_norm": 3.227780818939209,
+      "learning_rate": 2.2186346863468634e-06,
+      "loss": 1.3587,
+      "step": 1463
+    },
+    {
+      "epoch": 1.822028624766646,
+      "grad_norm": 3.0375192165374756,
+      "learning_rate": 2.2163284132841333e-06,
+      "loss": 1.4015,
+      "step": 1464
+    },
+    {
+      "epoch": 1.8232731798382078,
+      "grad_norm": 3.0200846195220947,
+      "learning_rate": 2.2140221402214023e-06,
+      "loss": 1.419,
+      "step": 1465
+    },
+    {
+      "epoch": 1.8245177349097697,
+      "grad_norm": 2.9456615447998047,
+      "learning_rate": 2.2117158671586717e-06,
+      "loss": 1.4075,
+      "step": 1466
+    },
+    {
+      "epoch": 1.8257622899813317,
+      "grad_norm": 3.3129165172576904,
+      "learning_rate": 2.209409594095941e-06,
+      "loss": 1.4241,
+      "step": 1467
+    },
+    {
+      "epoch": 1.8270068450528936,
+      "grad_norm": 3.013141393661499,
+      "learning_rate": 2.2071033210332106e-06,
+      "loss": 1.4066,
+      "step": 1468
+    },
+    {
+      "epoch": 1.8282514001244556,
+      "grad_norm": 2.8414366245269775,
+      "learning_rate": 2.20479704797048e-06,
+      "loss": 1.4009,
+      "step": 1469
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "grad_norm": 2.8920438289642334,
+      "learning_rate": 2.2024907749077494e-06,
+      "loss": 1.3916,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "eval_loss": 1.6220412254333496,
+      "eval_runtime": 52.6199,
+      "eval_samples_per_second": 19.004,
+      "eval_steps_per_second": 0.798,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8307405102675793,
+      "grad_norm": 3.032353639602661,
+      "learning_rate": 2.2001845018450184e-06,
+      "loss": 1.402,
+      "step": 1471
+    },
+    {
+      "epoch": 1.8319850653391412,
+      "grad_norm": 2.9512081146240234,
+      "learning_rate": 2.1978782287822883e-06,
+      "loss": 1.429,
+      "step": 1472
+    },
+    {
+      "epoch": 1.833229620410703,
+      "grad_norm": 2.846081018447876,
+      "learning_rate": 2.1955719557195573e-06,
+      "loss": 1.4203,
+      "step": 1473
+    },
+    {
+      "epoch": 1.834474175482265,
+      "grad_norm": 3.093378782272339,
+      "learning_rate": 2.1932656826568267e-06,
+      "loss": 1.4307,
+      "step": 1474
+    },
+    {
+      "epoch": 1.8357187305538272,
+      "grad_norm": 2.808326005935669,
+      "learning_rate": 2.190959409594096e-06,
+      "loss": 1.3753,
+      "step": 1475
+    },
+    {
+      "epoch": 1.836963285625389,
+      "grad_norm": 3.0328383445739746,
+      "learning_rate": 2.1886531365313655e-06,
+      "loss": 1.4534,
+      "step": 1476
+    },
+    {
+      "epoch": 1.8382078406969509,
+      "grad_norm": 3.045897960662842,
+      "learning_rate": 2.186346863468635e-06,
+      "loss": 1.3259,
+      "step": 1477
+    },
+    {
+      "epoch": 1.8394523957685127,
+      "grad_norm": 3.171846628189087,
+      "learning_rate": 2.1840405904059044e-06,
+      "loss": 1.397,
+      "step": 1478
+    },
+    {
+      "epoch": 1.8406969508400746,
+      "grad_norm": 2.893061637878418,
+      "learning_rate": 2.1817343173431734e-06,
+      "loss": 1.4201,
+      "step": 1479
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "grad_norm": 3.0151467323303223,
+      "learning_rate": 2.1794280442804432e-06,
+      "loss": 1.4266,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "eval_loss": 1.6166906356811523,
+      "eval_runtime": 51.8266,
+      "eval_samples_per_second": 19.295,
+      "eval_steps_per_second": 0.81,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8431860609831985,
+      "grad_norm": 2.942431688308716,
+      "learning_rate": 2.177121771217712e-06,
+      "loss": 1.3744,
+      "step": 1481
+    },
+    {
+      "epoch": 1.8444306160547606,
+      "grad_norm": 2.8482232093811035,
+      "learning_rate": 2.1748154981549816e-06,
+      "loss": 1.3914,
+      "step": 1482
+    },
+    {
+      "epoch": 1.8456751711263224,
+      "grad_norm": 2.9315359592437744,
+      "learning_rate": 2.172509225092251e-06,
+      "loss": 1.419,
+      "step": 1483
+    },
+    {
+      "epoch": 1.8469197261978842,
+      "grad_norm": 3.191800594329834,
+      "learning_rate": 2.1702029520295205e-06,
+      "loss": 1.3524,
+      "step": 1484
+    },
+    {
+      "epoch": 1.848164281269446,
+      "grad_norm": 2.9308040142059326,
+      "learning_rate": 2.16789667896679e-06,
+      "loss": 1.3742,
+      "step": 1485
+    },
+    {
+      "epoch": 1.849408836341008,
+      "grad_norm": 2.7678964138031006,
+      "learning_rate": 2.1655904059040593e-06,
+      "loss": 1.3786,
+      "step": 1486
+    },
+    {
+      "epoch": 1.85065339141257,
+      "grad_norm": 3.0300636291503906,
+      "learning_rate": 2.1632841328413287e-06,
+      "loss": 1.4469,
+      "step": 1487
+    },
+    {
+      "epoch": 1.851897946484132,
+      "grad_norm": 3.142781972885132,
+      "learning_rate": 2.160977859778598e-06,
+      "loss": 1.4565,
+      "step": 1488
+    },
+    {
+      "epoch": 1.853142501555694,
+      "grad_norm": 2.954916000366211,
+      "learning_rate": 2.158671586715867e-06,
+      "loss": 1.4356,
+      "step": 1489
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "grad_norm": 2.864875316619873,
+      "learning_rate": 2.1563653136531366e-06,
+      "loss": 1.4521,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "eval_loss": 1.6207302808761597,
+      "eval_runtime": 49.2215,
+      "eval_samples_per_second": 20.316,
+      "eval_steps_per_second": 0.853,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8556316116988176,
+      "grad_norm": 2.8908467292785645,
+      "learning_rate": 2.154059040590406e-06,
+      "loss": 1.4304,
+      "step": 1491
+    },
+    {
+      "epoch": 1.8568761667703795,
+      "grad_norm": 2.9439549446105957,
+      "learning_rate": 2.1517527675276754e-06,
+      "loss": 1.398,
+      "step": 1492
+    },
+    {
+      "epoch": 1.8581207218419415,
+      "grad_norm": 2.967728853225708,
+      "learning_rate": 2.149446494464945e-06,
+      "loss": 1.5038,
+      "step": 1493
+    },
+    {
+      "epoch": 1.8593652769135034,
+      "grad_norm": 2.938123941421509,
+      "learning_rate": 2.1471402214022143e-06,
+      "loss": 1.4395,
+      "step": 1494
+    },
+    {
+      "epoch": 1.8606098319850655,
+      "grad_norm": 2.93609881401062,
+      "learning_rate": 2.1448339483394837e-06,
+      "loss": 1.3939,
+      "step": 1495
+    },
+    {
+      "epoch": 1.8618543870566273,
+      "grad_norm": 2.9093775749206543,
+      "learning_rate": 2.142527675276753e-06,
+      "loss": 1.3942,
+      "step": 1496
+    },
+    {
+      "epoch": 1.8630989421281892,
+      "grad_norm": 2.773960590362549,
+      "learning_rate": 2.140221402214022e-06,
+      "loss": 1.3886,
+      "step": 1497
+    },
+    {
+      "epoch": 1.864343497199751,
+      "grad_norm": 2.9858205318450928,
+      "learning_rate": 2.1379151291512916e-06,
+      "loss": 1.3814,
+      "step": 1498
+    },
+    {
+      "epoch": 1.8655880522713129,
+      "grad_norm": 3.1233417987823486,
+      "learning_rate": 2.135608856088561e-06,
+      "loss": 1.4313,
+      "step": 1499
+    },
+    {
+      "epoch": 1.866832607342875,
+      "grad_norm": 2.9569480419158936,
+      "learning_rate": 2.1333025830258304e-06,
+      "loss": 1.4302,
+      "step": 1500
+    },
+    {
+      "epoch": 1.866832607342875,
+      "eval_loss": 1.6130231618881226,
+      "eval_runtime": 46.2834,
+      "eval_samples_per_second": 21.606,
+      "eval_steps_per_second": 0.907,
+      "step": 1500
+    },
+    {
+      "epoch": 1.868077162414437,
+      "grad_norm": 2.926203727722168,
+      "learning_rate": 2.1309963099631e-06,
+      "loss": 1.4236,
+      "step": 1501
+    },
+    {
+      "epoch": 1.8693217174859988,
+      "grad_norm": 2.9496357440948486,
+      "learning_rate": 2.1286900369003692e-06,
+      "loss": 1.4014,
+      "step": 1502
+    },
+    {
+      "epoch": 1.8705662725575607,
+      "grad_norm": 2.9314279556274414,
+      "learning_rate": 2.1263837638376387e-06,
+      "loss": 1.4414,
+      "step": 1503
+    },
+    {
+      "epoch": 1.8718108276291225,
+      "grad_norm": 2.889460802078247,
+      "learning_rate": 2.124077490774908e-06,
+      "loss": 1.3883,
+      "step": 1504
+    },
+    {
+      "epoch": 1.8730553827006844,
+      "grad_norm": 2.9859671592712402,
+      "learning_rate": 2.1217712177121775e-06,
+      "loss": 1.4551,
+      "step": 1505
+    },
+    {
+      "epoch": 1.8742999377722465,
+      "grad_norm": 2.9330570697784424,
+      "learning_rate": 2.1194649446494465e-06,
+      "loss": 1.3895,
+      "step": 1506
+    },
+    {
+      "epoch": 1.8755444928438083,
+      "grad_norm": 2.9062387943267822,
+      "learning_rate": 2.117158671586716e-06,
+      "loss": 1.4492,
+      "step": 1507
+    },
+    {
+      "epoch": 1.8767890479153704,
+      "grad_norm": 3.0337791442871094,
+      "learning_rate": 2.1148523985239854e-06,
+      "loss": 1.4682,
+      "step": 1508
+    },
+    {
+      "epoch": 1.8780336029869322,
+      "grad_norm": 3.0448431968688965,
+      "learning_rate": 2.1125461254612548e-06,
+      "loss": 1.4836,
+      "step": 1509
+    },
+    {
+      "epoch": 1.879278158058494,
+      "grad_norm": 2.912764310836792,
+      "learning_rate": 2.110239852398524e-06,
+      "loss": 1.4644,
+      "step": 1510
+    },
+    {
+      "epoch": 1.879278158058494,
+      "eval_loss": 1.6124104261398315,
+      "eval_runtime": 41.9713,
+      "eval_samples_per_second": 23.826,
+      "eval_steps_per_second": 1.001,
+      "step": 1510
+    },
+    {
+      "epoch": 1.880522713130056,
+      "grad_norm": 3.0399203300476074,
+      "learning_rate": 2.1079335793357936e-06,
+      "loss": 1.4391,
+      "step": 1511
+    },
+    {
+      "epoch": 1.8817672682016178,
+      "grad_norm": 2.9852235317230225,
+      "learning_rate": 2.1056273062730626e-06,
+      "loss": 1.346,
+      "step": 1512
+    },
+    {
+      "epoch": 1.8830118232731798,
+      "grad_norm": 2.904437780380249,
+      "learning_rate": 2.1033210332103325e-06,
+      "loss": 1.4523,
+      "step": 1513
+    },
+    {
+      "epoch": 1.884256378344742,
+      "grad_norm": 2.8988049030303955,
+      "learning_rate": 2.1010147601476015e-06,
+      "loss": 1.4001,
+      "step": 1514
+    },
+    {
+      "epoch": 1.8855009334163038,
+      "grad_norm": 2.9890260696411133,
+      "learning_rate": 2.098708487084871e-06,
+      "loss": 1.4074,
+      "step": 1515
+    },
+    {
+      "epoch": 1.8867454884878656,
+      "grad_norm": 3.0486819744110107,
+      "learning_rate": 2.0964022140221403e-06,
+      "loss": 1.461,
+      "step": 1516
+    },
+    {
+      "epoch": 1.8879900435594275,
+      "grad_norm": 2.8746016025543213,
+      "learning_rate": 2.0940959409594097e-06,
+      "loss": 1.4013,
+      "step": 1517
+    },
+    {
+      "epoch": 1.8892345986309893,
+      "grad_norm": 2.979480266571045,
+      "learning_rate": 2.091789667896679e-06,
+      "loss": 1.4567,
+      "step": 1518
+    },
+    {
+      "epoch": 1.8904791537025514,
+      "grad_norm": 2.8351497650146484,
+      "learning_rate": 2.0894833948339486e-06,
+      "loss": 1.3655,
+      "step": 1519
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "grad_norm": 2.9061646461486816,
+      "learning_rate": 2.0871771217712176e-06,
+      "loss": 1.4165,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "eval_loss": 1.6172568798065186,
+      "eval_runtime": 42.998,
+      "eval_samples_per_second": 23.257,
+      "eval_steps_per_second": 0.977,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8929682638456753,
+      "grad_norm": 2.8751838207244873,
+      "learning_rate": 2.0848708487084874e-06,
+      "loss": 1.4133,
+      "step": 1521
+    },
+    {
+      "epoch": 1.8942128189172371,
+      "grad_norm": 2.7513587474823,
+      "learning_rate": 2.0825645756457564e-06,
+      "loss": 1.3734,
+      "step": 1522
+    },
+    {
+      "epoch": 1.895457373988799,
+      "grad_norm": 2.9244418144226074,
+      "learning_rate": 2.0802583025830263e-06,
+      "loss": 1.4181,
+      "step": 1523
+    },
+    {
+      "epoch": 1.8967019290603608,
+      "grad_norm": 2.877495765686035,
+      "learning_rate": 2.0779520295202953e-06,
+      "loss": 1.4009,
+      "step": 1524
+    },
+    {
+      "epoch": 1.8979464841319227,
+      "grad_norm": 2.9316184520721436,
+      "learning_rate": 2.0756457564575647e-06,
+      "loss": 1.3796,
+      "step": 1525
+    },
+    {
+      "epoch": 1.8991910392034848,
+      "grad_norm": 3.0403976440429688,
+      "learning_rate": 2.073339483394834e-06,
+      "loss": 1.4101,
+      "step": 1526
+    },
+    {
+      "epoch": 1.9004355942750468,
+      "grad_norm": 3.0020930767059326,
+      "learning_rate": 2.0710332103321035e-06,
+      "loss": 1.4166,
+      "step": 1527
+    },
+    {
+      "epoch": 1.9016801493466087,
+      "grad_norm": 3.0937304496765137,
+      "learning_rate": 2.0687269372693725e-06,
+      "loss": 1.4599,
+      "step": 1528
+    },
+    {
+      "epoch": 1.9029247044181705,
+      "grad_norm": 2.901540517807007,
+      "learning_rate": 2.0664206642066424e-06,
+      "loss": 1.3642,
+      "step": 1529
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "grad_norm": 3.0517938137054443,
+      "learning_rate": 2.0641143911439114e-06,
+      "loss": 1.451,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "eval_loss": 1.6178728342056274,
+      "eval_runtime": 43.9067,
+      "eval_samples_per_second": 22.776,
+      "eval_steps_per_second": 0.957,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9054138145612942,
+      "grad_norm": 3.12484073638916,
+      "learning_rate": 2.0618081180811812e-06,
+      "loss": 1.4522,
+      "step": 1531
+    },
+    {
+      "epoch": 1.9066583696328563,
+      "grad_norm": 2.9447414875030518,
+      "learning_rate": 2.0595018450184502e-06,
+      "loss": 1.4154,
+      "step": 1532
+    },
+    {
+      "epoch": 1.9079029247044181,
+      "grad_norm": 2.909738779067993,
+      "learning_rate": 2.0571955719557197e-06,
+      "loss": 1.4133,
+      "step": 1533
+    },
+    {
+      "epoch": 1.9091474797759802,
+      "grad_norm": 3.1522421836853027,
+      "learning_rate": 2.054889298892989e-06,
+      "loss": 1.4777,
+      "step": 1534
+    },
+    {
+      "epoch": 1.910392034847542,
+      "grad_norm": 3.041447639465332,
+      "learning_rate": 2.0525830258302585e-06,
+      "loss": 1.4104,
+      "step": 1535
+    },
+    {
+      "epoch": 1.911636589919104,
+      "grad_norm": 3.01129412651062,
+      "learning_rate": 2.050276752767528e-06,
+      "loss": 1.3852,
+      "step": 1536
+    },
+    {
+      "epoch": 1.9128811449906657,
+      "grad_norm": 3.0697543621063232,
+      "learning_rate": 2.0479704797047974e-06,
+      "loss": 1.4389,
+      "step": 1537
+    },
+    {
+      "epoch": 1.9141257000622276,
+      "grad_norm": 2.9391236305236816,
+      "learning_rate": 2.0456642066420664e-06,
+      "loss": 1.4239,
+      "step": 1538
+    },
+    {
+      "epoch": 1.9153702551337897,
+      "grad_norm": 2.978236675262451,
+      "learning_rate": 2.043357933579336e-06,
+      "loss": 1.462,
+      "step": 1539
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "grad_norm": 2.8124167919158936,
+      "learning_rate": 2.041051660516605e-06,
+      "loss": 1.3186,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "eval_loss": 1.6120468378067017,
+      "eval_runtime": 43.3787,
+      "eval_samples_per_second": 23.053,
+      "eval_steps_per_second": 0.968,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9178593652769136,
+      "grad_norm": 3.029939889907837,
+      "learning_rate": 2.0387453874538746e-06,
+      "loss": 1.4481,
+      "step": 1541
+    },
+    {
+      "epoch": 1.9191039203484754,
+      "grad_norm": 2.8749356269836426,
+      "learning_rate": 2.036439114391144e-06,
+      "loss": 1.3735,
+      "step": 1542
+    },
+    {
+      "epoch": 1.9203484754200373,
+      "grad_norm": 2.9181101322174072,
+      "learning_rate": 2.0341328413284135e-06,
+      "loss": 1.4085,
+      "step": 1543
+    },
+    {
+      "epoch": 1.9215930304915991,
+      "grad_norm": 2.936319589614868,
+      "learning_rate": 2.031826568265683e-06,
+      "loss": 1.3827,
+      "step": 1544
+    },
+    {
+      "epoch": 1.9228375855631612,
+      "grad_norm": 2.9286844730377197,
+      "learning_rate": 2.0295202952029523e-06,
+      "loss": 1.4563,
+      "step": 1545
+    },
+    {
+      "epoch": 1.924082140634723,
+      "grad_norm": 2.790752649307251,
+      "learning_rate": 2.0272140221402213e-06,
+      "loss": 1.3935,
+      "step": 1546
+    },
+    {
+      "epoch": 1.9253266957062851,
+      "grad_norm": 2.878164529800415,
+      "learning_rate": 2.024907749077491e-06,
+      "loss": 1.4159,
+      "step": 1547
+    },
+    {
+      "epoch": 1.926571250777847,
+      "grad_norm": 3.0583767890930176,
+      "learning_rate": 2.02260147601476e-06,
+      "loss": 1.4036,
+      "step": 1548
+    },
+    {
+      "epoch": 1.9278158058494088,
+      "grad_norm": 2.943686008453369,
+      "learning_rate": 2.02029520295203e-06,
+      "loss": 1.4178,
+      "step": 1549
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "grad_norm": 2.983553886413574,
+      "learning_rate": 2.017988929889299e-06,
+      "loss": 1.3959,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "eval_loss": 1.610864520072937,
+      "eval_runtime": 44.5262,
+      "eval_samples_per_second": 22.459,
+      "eval_steps_per_second": 0.943,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9303049159925325,
+      "grad_norm": 3.0323703289031982,
+      "learning_rate": 2.0156826568265684e-06,
+      "loss": 1.4362,
+      "step": 1551
+    },
+    {
+      "epoch": 1.9315494710640946,
+      "grad_norm": 3.036627769470215,
+      "learning_rate": 2.013376383763838e-06,
+      "loss": 1.4132,
+      "step": 1552
+    },
+    {
+      "epoch": 1.9327940261356567,
+      "grad_norm": 2.901071310043335,
+      "learning_rate": 2.0110701107011073e-06,
+      "loss": 1.3846,
+      "step": 1553
+    },
+    {
+      "epoch": 1.9340385812072185,
+      "grad_norm": 3.002803325653076,
+      "learning_rate": 2.0087638376383767e-06,
+      "loss": 1.4402,
+      "step": 1554
+    },
+    {
+      "epoch": 1.9352831362787803,
+      "grad_norm": 2.969228982925415,
+      "learning_rate": 2.006457564575646e-06,
+      "loss": 1.4387,
+      "step": 1555
+    },
+    {
+      "epoch": 1.9365276913503422,
+      "grad_norm": 2.9773998260498047,
+      "learning_rate": 2.004151291512915e-06,
+      "loss": 1.4149,
+      "step": 1556
+    },
+    {
+      "epoch": 1.937772246421904,
+      "grad_norm": 2.831655502319336,
+      "learning_rate": 2.0018450184501845e-06,
+      "loss": 1.3753,
+      "step": 1557
+    },
+    {
+      "epoch": 1.9390168014934661,
+      "grad_norm": 2.835994005203247,
+      "learning_rate": 1.999538745387454e-06,
+      "loss": 1.345,
+      "step": 1558
+    },
+    {
+      "epoch": 1.940261356565028,
+      "grad_norm": 3.0328211784362793,
+      "learning_rate": 1.9972324723247234e-06,
+      "loss": 1.3473,
+      "step": 1559
+    },
+    {
+      "epoch": 1.94150591163659,
+      "grad_norm": 2.9441378116607666,
+      "learning_rate": 1.994926199261993e-06,
+      "loss": 1.3913,
+      "step": 1560
+    },
+    {
+      "epoch": 1.94150591163659,
+      "eval_loss": 1.6193854808807373,
+      "eval_runtime": 44.6194,
+      "eval_samples_per_second": 22.412,
+      "eval_steps_per_second": 0.941,
+      "step": 1560
+    },
+    {
+      "epoch": 1.9427504667081519,
+      "grad_norm": 2.905486822128296,
+      "learning_rate": 1.9926199261992622e-06,
+      "loss": 1.4043,
+      "step": 1561
+    },
+    {
+      "epoch": 1.9439950217797137,
+      "grad_norm": 2.7758026123046875,
+      "learning_rate": 1.9903136531365317e-06,
+      "loss": 1.3974,
+      "step": 1562
+    },
+    {
+      "epoch": 1.9452395768512756,
+      "grad_norm": 2.7883355617523193,
+      "learning_rate": 1.988007380073801e-06,
+      "loss": 1.4246,
+      "step": 1563
+    },
+    {
+      "epoch": 1.9464841319228374,
+      "grad_norm": 2.8322808742523193,
+      "learning_rate": 1.98570110701107e-06,
+      "loss": 1.4002,
+      "step": 1564
+    },
+    {
+      "epoch": 1.9477286869943995,
+      "grad_norm": 2.948850631713867,
+      "learning_rate": 1.9833948339483395e-06,
+      "loss": 1.3866,
+      "step": 1565
+    },
+    {
+      "epoch": 1.9489732420659616,
+      "grad_norm": 2.9772586822509766,
+      "learning_rate": 1.981088560885609e-06,
+      "loss": 1.4473,
+      "step": 1566
+    },
+    {
+      "epoch": 1.9502177971375234,
+      "grad_norm": 2.9183740615844727,
+      "learning_rate": 1.9787822878228783e-06,
+      "loss": 1.3943,
+      "step": 1567
+    },
+    {
+      "epoch": 1.9514623522090853,
+      "grad_norm": 3.10422682762146,
+      "learning_rate": 1.9764760147601478e-06,
+      "loss": 1.465,
+      "step": 1568
+    },
+    {
+      "epoch": 1.952706907280647,
+      "grad_norm": 3.001533031463623,
+      "learning_rate": 1.974169741697417e-06,
+      "loss": 1.4133,
+      "step": 1569
+    },
+    {
+      "epoch": 1.953951462352209,
+      "grad_norm": 2.9521870613098145,
+      "learning_rate": 1.9718634686346866e-06,
+      "loss": 1.4652,
+      "step": 1570
+    },
+    {
+      "epoch": 1.953951462352209,
+      "eval_loss": 1.617977499961853,
+      "eval_runtime": 45.7189,
+      "eval_samples_per_second": 21.873,
+      "eval_steps_per_second": 0.919,
+      "step": 1570
+    },
+    {
+      "epoch": 1.955196017423771,
+      "grad_norm": 2.9504826068878174,
+      "learning_rate": 1.969557195571956e-06,
+      "loss": 1.4117,
+      "step": 1571
+    },
+    {
+      "epoch": 1.9564405724953329,
+      "grad_norm": 2.9383559226989746,
+      "learning_rate": 1.9672509225092255e-06,
+      "loss": 1.3439,
+      "step": 1572
+    },
+    {
+      "epoch": 1.957685127566895,
+      "grad_norm": 2.850644588470459,
+      "learning_rate": 1.9649446494464945e-06,
+      "loss": 1.3521,
+      "step": 1573
+    },
+    {
+      "epoch": 1.9589296826384568,
+      "grad_norm": 2.8787410259246826,
+      "learning_rate": 1.962638376383764e-06,
+      "loss": 1.4268,
+      "step": 1574
+    },
+    {
+      "epoch": 1.9601742377100186,
+      "grad_norm": 2.966080665588379,
+      "learning_rate": 1.9603321033210333e-06,
+      "loss": 1.3367,
+      "step": 1575
+    },
+    {
+      "epoch": 1.9614187927815805,
+      "grad_norm": 2.858222007751465,
+      "learning_rate": 1.9580258302583027e-06,
+      "loss": 1.3842,
+      "step": 1576
+    },
+    {
+      "epoch": 1.9626633478531423,
+      "grad_norm": 2.924525737762451,
+      "learning_rate": 1.955719557195572e-06,
+      "loss": 1.4561,
+      "step": 1577
+    },
+    {
+      "epoch": 1.9639079029247044,
+      "grad_norm": 2.9281182289123535,
+      "learning_rate": 1.9534132841328416e-06,
+      "loss": 1.4179,
+      "step": 1578
+    },
+    {
+      "epoch": 1.9651524579962665,
+      "grad_norm": 2.982023000717163,
+      "learning_rate": 1.951107011070111e-06,
+      "loss": 1.4413,
+      "step": 1579
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "grad_norm": 2.7912652492523193,
+      "learning_rate": 1.9488007380073804e-06,
+      "loss": 1.3577,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "eval_loss": 1.6062211990356445,
+      "eval_runtime": 46.7931,
+      "eval_samples_per_second": 21.371,
+      "eval_steps_per_second": 0.898,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9676415681393902,
+      "grad_norm": 2.9585583209991455,
+      "learning_rate": 1.9464944649446494e-06,
+      "loss": 1.4084,
+      "step": 1581
+    },
+    {
+      "epoch": 1.968886123210952,
+      "grad_norm": 2.879201889038086,
+      "learning_rate": 1.944188191881919e-06,
+      "loss": 1.4164,
+      "step": 1582
+    },
+    {
+      "epoch": 1.9701306782825139,
+      "grad_norm": 2.9510116577148438,
+      "learning_rate": 1.9418819188191883e-06,
+      "loss": 1.4211,
+      "step": 1583
+    },
+    {
+      "epoch": 1.971375233354076,
+      "grad_norm": 2.973844528198242,
+      "learning_rate": 1.9395756457564577e-06,
+      "loss": 1.4358,
+      "step": 1584
+    },
+    {
+      "epoch": 1.9726197884256378,
+      "grad_norm": 2.939131736755371,
+      "learning_rate": 1.937269372693727e-06,
+      "loss": 1.407,
+      "step": 1585
+    },
+    {
+      "epoch": 1.9738643434971999,
+      "grad_norm": 2.956646680831909,
+      "learning_rate": 1.9349630996309965e-06,
+      "loss": 1.3872,
+      "step": 1586
+    },
+    {
+      "epoch": 1.9751088985687617,
+      "grad_norm": 3.070354461669922,
+      "learning_rate": 1.932656826568266e-06,
+      "loss": 1.412,
+      "step": 1587
+    },
+    {
+      "epoch": 1.9763534536403236,
+      "grad_norm": 2.8143749237060547,
+      "learning_rate": 1.9303505535055354e-06,
+      "loss": 1.3522,
+      "step": 1588
+    },
+    {
+      "epoch": 1.9775980087118854,
+      "grad_norm": 2.8610002994537354,
+      "learning_rate": 1.9280442804428044e-06,
+      "loss": 1.3988,
+      "step": 1589
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "grad_norm": 2.951023817062378,
+      "learning_rate": 1.925738007380074e-06,
+      "loss": 1.3855,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "eval_loss": 1.613527536392212,
+      "eval_runtime": 46.5242,
+      "eval_samples_per_second": 21.494,
+      "eval_steps_per_second": 0.903,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9800871188550093,
+      "grad_norm": 2.9333510398864746,
+      "learning_rate": 1.9234317343173432e-06,
+      "loss": 1.4518,
+      "step": 1591
+    },
+    {
+      "epoch": 1.9813316739265714,
+      "grad_norm": 2.847015619277954,
+      "learning_rate": 1.9211254612546126e-06,
+      "loss": 1.3999,
+      "step": 1592
+    },
+    {
+      "epoch": 1.9825762289981332,
+      "grad_norm": 2.969320058822632,
+      "learning_rate": 1.918819188191882e-06,
+      "loss": 1.4241,
+      "step": 1593
+    },
+    {
+      "epoch": 1.983820784069695,
+      "grad_norm": 2.862797498703003,
+      "learning_rate": 1.9165129151291515e-06,
+      "loss": 1.3929,
+      "step": 1594
+    },
+    {
+      "epoch": 1.985065339141257,
+      "grad_norm": 2.90384840965271,
+      "learning_rate": 1.914206642066421e-06,
+      "loss": 1.4127,
+      "step": 1595
+    },
+    {
+      "epoch": 1.9863098942128188,
+      "grad_norm": 2.8059256076812744,
+      "learning_rate": 1.9119003690036903e-06,
+      "loss": 1.4155,
+      "step": 1596
+    },
+    {
+      "epoch": 1.9875544492843809,
+      "grad_norm": 2.9068455696105957,
+      "learning_rate": 1.9095940959409593e-06,
+      "loss": 1.3924,
+      "step": 1597
+    },
+    {
+      "epoch": 1.9887990043559427,
+      "grad_norm": 2.903542995452881,
+      "learning_rate": 1.907287822878229e-06,
+      "loss": 1.4039,
+      "step": 1598
+    },
+    {
+      "epoch": 1.9900435594275048,
+      "grad_norm": 2.8566174507141113,
+      "learning_rate": 1.9049815498154984e-06,
+      "loss": 1.4276,
+      "step": 1599
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "grad_norm": 2.8891515731811523,
+      "learning_rate": 1.9026752767527676e-06,
+      "loss": 1.4555,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "eval_loss": 1.6086686849594116,
+      "eval_runtime": 46.2785,
+      "eval_samples_per_second": 21.608,
+      "eval_steps_per_second": 0.908,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9925326695706285,
+      "grad_norm": 2.933851957321167,
+      "learning_rate": 1.900369003690037e-06,
+      "loss": 1.3881,
+      "step": 1601
+    },
+    {
+      "epoch": 1.9937772246421903,
+      "grad_norm": 2.8393521308898926,
+      "learning_rate": 1.8980627306273065e-06,
+      "loss": 1.3959,
+      "step": 1602
+    },
+    {
+      "epoch": 1.9950217797137524,
+      "grad_norm": 2.839740753173828,
+      "learning_rate": 1.8957564575645759e-06,
+      "loss": 1.384,
+      "step": 1603
+    },
+    {
+      "epoch": 1.9962663347853142,
+      "grad_norm": 2.7918097972869873,
+      "learning_rate": 1.8934501845018453e-06,
+      "loss": 1.4173,
+      "step": 1604
+    },
+    {
+      "epoch": 1.9975108898568763,
+      "grad_norm": 2.920839548110962,
+      "learning_rate": 1.8911439114391145e-06,
+      "loss": 1.3693,
+      "step": 1605
+    },
+    {
+      "epoch": 1.9987554449284382,
+      "grad_norm": 2.9545562267303467,
+      "learning_rate": 1.888837638376384e-06,
+      "loss": 1.4062,
+      "step": 1606
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 2.912815570831299,
+      "learning_rate": 1.8865313653136534e-06,
+      "loss": 1.4146,
+      "step": 1607
+    },
+    {
+      "epoch": 2.001244555071562,
+      "grad_norm": 3.1125903129577637,
+      "learning_rate": 1.8842250922509226e-06,
+      "loss": 1.3294,
+      "step": 1608
+    },
+    {
+      "epoch": 2.0024891101431237,
+      "grad_norm": 3.123905658721924,
+      "learning_rate": 1.881918819188192e-06,
+      "loss": 1.4175,
+      "step": 1609
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "grad_norm": 3.055100440979004,
+      "learning_rate": 1.8796125461254614e-06,
+      "loss": 1.3628,
+      "step": 1610
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "eval_loss": 1.6198724508285522,
+      "eval_runtime": 42.5155,
+      "eval_samples_per_second": 23.521,
+      "eval_steps_per_second": 0.988,
+      "step": 1610
+    },
+    {
+      "epoch": 2.004978220286248,
+      "grad_norm": 3.087313413619995,
+      "learning_rate": 1.8773062730627308e-06,
+      "loss": 1.292,
+      "step": 1611
+    },
+    {
+      "epoch": 2.0062227753578097,
+      "grad_norm": 3.1849849224090576,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 1.3393,
+      "step": 1612
+    },
+    {
+      "epoch": 2.0074673304293715,
+      "grad_norm": 3.243750810623169,
+      "learning_rate": 1.8726937269372695e-06,
+      "loss": 1.3367,
+      "step": 1613
+    },
+    {
+      "epoch": 2.0087118855009334,
+      "grad_norm": 3.3086085319519043,
+      "learning_rate": 1.8703874538745389e-06,
+      "loss": 1.3099,
+      "step": 1614
+    },
+    {
+      "epoch": 2.0099564405724952,
+      "grad_norm": 3.388633966445923,
+      "learning_rate": 1.8680811808118083e-06,
+      "loss": 1.3816,
+      "step": 1615
+    },
+    {
+      "epoch": 2.011200995644057,
+      "grad_norm": 3.422671318054199,
+      "learning_rate": 1.8657749077490777e-06,
+      "loss": 1.3799,
+      "step": 1616
+    },
+    {
+      "epoch": 2.0124455507156194,
+      "grad_norm": 3.1104466915130615,
+      "learning_rate": 1.863468634686347e-06,
+      "loss": 1.3752,
+      "step": 1617
+    },
+    {
+      "epoch": 2.013690105787181,
+      "grad_norm": 3.2728488445281982,
+      "learning_rate": 1.8611623616236164e-06,
+      "loss": 1.322,
+      "step": 1618
+    },
+    {
+      "epoch": 2.014934660858743,
+      "grad_norm": 3.3357644081115723,
+      "learning_rate": 1.8588560885608858e-06,
+      "loss": 1.3193,
+      "step": 1619
+    },
+    {
+      "epoch": 2.016179215930305,
+      "grad_norm": 3.1026456356048584,
+      "learning_rate": 1.856549815498155e-06,
+      "loss": 1.3394,
+      "step": 1620
+    },
+    {
+      "epoch": 2.016179215930305,
+      "eval_loss": 1.6217585802078247,
+      "eval_runtime": 45.9794,
+      "eval_samples_per_second": 21.749,
+      "eval_steps_per_second": 0.913,
+      "step": 1620
+    },
+    {
+      "epoch": 2.0174237710018668,
+      "grad_norm": 3.314291000366211,
+      "learning_rate": 1.8542435424354244e-06,
+      "loss": 1.3778,
+      "step": 1621
+    },
+    {
+      "epoch": 2.0186683260734286,
+      "grad_norm": 3.3624770641326904,
+      "learning_rate": 1.8519372693726939e-06,
+      "loss": 1.3948,
+      "step": 1622
+    },
+    {
+      "epoch": 2.0199128811449905,
+      "grad_norm": 3.4041574001312256,
+      "learning_rate": 1.8496309963099633e-06,
+      "loss": 1.3109,
+      "step": 1623
+    },
+    {
+      "epoch": 2.0211574362165527,
+      "grad_norm": 3.5128726959228516,
+      "learning_rate": 1.8473247232472327e-06,
+      "loss": 1.4223,
+      "step": 1624
+    },
+    {
+      "epoch": 2.0224019912881146,
+      "grad_norm": 3.444768190383911,
+      "learning_rate": 1.845018450184502e-06,
+      "loss": 1.3672,
+      "step": 1625
+    },
+    {
+      "epoch": 2.0236465463596764,
+      "grad_norm": 3.478060722351074,
+      "learning_rate": 1.8427121771217713e-06,
+      "loss": 1.4384,
+      "step": 1626
+    },
+    {
+      "epoch": 2.0248911014312383,
+      "grad_norm": 3.211465835571289,
+      "learning_rate": 1.8404059040590408e-06,
+      "loss": 1.3891,
+      "step": 1627
+    },
+    {
+      "epoch": 2.0261356565028,
+      "grad_norm": 3.1283977031707764,
+      "learning_rate": 1.83809963099631e-06,
+      "loss": 1.3752,
+      "step": 1628
+    },
+    {
+      "epoch": 2.027380211574362,
+      "grad_norm": 3.3699934482574463,
+      "learning_rate": 1.8357933579335796e-06,
+      "loss": 1.3751,
+      "step": 1629
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "grad_norm": 3.0729823112487793,
+      "learning_rate": 1.8334870848708488e-06,
+      "loss": 1.3452,
+      "step": 1630
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "eval_loss": 1.6185853481292725,
+      "eval_runtime": 45.2873,
+      "eval_samples_per_second": 22.081,
+      "eval_steps_per_second": 0.927,
+      "step": 1630
+    },
+    {
+      "epoch": 2.029869321717486,
+      "grad_norm": 3.274139642715454,
+      "learning_rate": 1.8311808118081182e-06,
+      "loss": 1.3242,
+      "step": 1631
+    },
+    {
+      "epoch": 2.031113876789048,
+      "grad_norm": 3.2869648933410645,
+      "learning_rate": 1.8288745387453877e-06,
+      "loss": 1.3435,
+      "step": 1632
+    },
+    {
+      "epoch": 2.03235843186061,
+      "grad_norm": 3.3614306449890137,
+      "learning_rate": 1.8265682656826569e-06,
+      "loss": 1.3705,
+      "step": 1633
+    },
+    {
+      "epoch": 2.0336029869321717,
+      "grad_norm": 3.411914587020874,
+      "learning_rate": 1.8242619926199265e-06,
+      "loss": 1.3942,
+      "step": 1634
+    },
+    {
+      "epoch": 2.0348475420037335,
+      "grad_norm": 3.274519920349121,
+      "learning_rate": 1.8219557195571957e-06,
+      "loss": 1.3393,
+      "step": 1635
+    },
+    {
+      "epoch": 2.0360920970752954,
+      "grad_norm": 3.993478775024414,
+      "learning_rate": 1.819649446494465e-06,
+      "loss": 1.3713,
+      "step": 1636
+    },
+    {
+      "epoch": 2.0373366521468577,
+      "grad_norm": 3.273571491241455,
+      "learning_rate": 1.8173431734317346e-06,
+      "loss": 1.3337,
+      "step": 1637
+    },
+    {
+      "epoch": 2.0385812072184195,
+      "grad_norm": 3.1542446613311768,
+      "learning_rate": 1.8150369003690038e-06,
+      "loss": 1.3793,
+      "step": 1638
+    },
+    {
+      "epoch": 2.0398257622899814,
+      "grad_norm": 3.263773202896118,
+      "learning_rate": 1.812730627306273e-06,
+      "loss": 1.3844,
+      "step": 1639
+    },
+    {
+      "epoch": 2.041070317361543,
+      "grad_norm": 3.268772840499878,
+      "learning_rate": 1.8104243542435426e-06,
+      "loss": 1.3947,
+      "step": 1640
+    },
+    {
+      "epoch": 2.041070317361543,
+      "eval_loss": 1.6149734258651733,
+      "eval_runtime": 48.5557,
+      "eval_samples_per_second": 20.595,
+      "eval_steps_per_second": 0.865,
+      "step": 1640
+    },
+    {
+      "epoch": 2.042314872433105,
+      "grad_norm": 3.190168619155884,
+      "learning_rate": 1.8081180811808118e-06,
+      "loss": 1.3276,
+      "step": 1641
+    },
+    {
+      "epoch": 2.043559427504667,
+      "grad_norm": 3.1573212146759033,
+      "learning_rate": 1.8058118081180815e-06,
+      "loss": 1.3307,
+      "step": 1642
+    },
+    {
+      "epoch": 2.044803982576229,
+      "grad_norm": 3.286513328552246,
+      "learning_rate": 1.8035055350553507e-06,
+      "loss": 1.3359,
+      "step": 1643
+    },
+    {
+      "epoch": 2.046048537647791,
+      "grad_norm": 3.342024803161621,
+      "learning_rate": 1.8011992619926199e-06,
+      "loss": 1.3472,
+      "step": 1644
+    },
+    {
+      "epoch": 2.047293092719353,
+      "grad_norm": 3.3237056732177734,
+      "learning_rate": 1.7988929889298895e-06,
+      "loss": 1.3898,
+      "step": 1645
+    },
+    {
+      "epoch": 2.0485376477909147,
+      "grad_norm": 3.370035409927368,
+      "learning_rate": 1.7965867158671587e-06,
+      "loss": 1.3537,
+      "step": 1646
+    },
+    {
+      "epoch": 2.0497822028624766,
+      "grad_norm": 3.1636598110198975,
+      "learning_rate": 1.7942804428044284e-06,
+      "loss": 1.3455,
+      "step": 1647
+    },
+    {
+      "epoch": 2.0510267579340384,
+      "grad_norm": 3.239396810531616,
+      "learning_rate": 1.7919741697416976e-06,
+      "loss": 1.3212,
+      "step": 1648
+    },
+    {
+      "epoch": 2.0522713130056003,
+      "grad_norm": 3.2765328884124756,
+      "learning_rate": 1.7896678966789668e-06,
+      "loss": 1.396,
+      "step": 1649
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "grad_norm": 3.2259459495544434,
+      "learning_rate": 1.7873616236162364e-06,
+      "loss": 1.3614,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "eval_loss": 1.614806056022644,
+      "eval_runtime": 49.8704,
+      "eval_samples_per_second": 20.052,
+      "eval_steps_per_second": 0.842,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0547604231487244,
+      "grad_norm": 3.257683277130127,
+      "learning_rate": 1.7850553505535056e-06,
+      "loss": 1.3663,
+      "step": 1651
+    },
+    {
+      "epoch": 2.0560049782202863,
+      "grad_norm": 3.292070150375366,
+      "learning_rate": 1.7827490774907748e-06,
+      "loss": 1.3548,
+      "step": 1652
+    },
+    {
+      "epoch": 2.057249533291848,
+      "grad_norm": 3.3517003059387207,
+      "learning_rate": 1.7804428044280445e-06,
+      "loss": 1.4015,
+      "step": 1653
+    },
+    {
+      "epoch": 2.05849408836341,
+      "grad_norm": 3.5100908279418945,
+      "learning_rate": 1.7781365313653137e-06,
+      "loss": 1.3666,
+      "step": 1654
+    },
+    {
+      "epoch": 2.059738643434972,
+      "grad_norm": 3.3054633140563965,
+      "learning_rate": 1.7758302583025833e-06,
+      "loss": 1.3691,
+      "step": 1655
+    },
+    {
+      "epoch": 2.060983198506534,
+      "grad_norm": 3.1036577224731445,
+      "learning_rate": 1.7735239852398525e-06,
+      "loss": 1.2721,
+      "step": 1656
+    },
+    {
+      "epoch": 2.062227753578096,
+      "grad_norm": 3.604623317718506,
+      "learning_rate": 1.7712177121771217e-06,
+      "loss": 1.376,
+      "step": 1657
+    },
+    {
+      "epoch": 2.063472308649658,
+      "grad_norm": 3.5246458053588867,
+      "learning_rate": 1.7689114391143914e-06,
+      "loss": 1.3891,
+      "step": 1658
+    },
+    {
+      "epoch": 2.0647168637212197,
+      "grad_norm": 3.286940336227417,
+      "learning_rate": 1.7666051660516606e-06,
+      "loss": 1.3978,
+      "step": 1659
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "grad_norm": 3.1022047996520996,
+      "learning_rate": 1.7642988929889302e-06,
+      "loss": 1.3373,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "eval_loss": 1.6144907474517822,
+      "eval_runtime": 48.4382,
+      "eval_samples_per_second": 20.645,
+      "eval_steps_per_second": 0.867,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0672059738643433,
+      "grad_norm": 3.4111568927764893,
+      "learning_rate": 1.7619926199261994e-06,
+      "loss": 1.3816,
+      "step": 1661
+    },
+    {
+      "epoch": 2.068450528935905,
+      "grad_norm": 3.478947162628174,
+      "learning_rate": 1.7596863468634687e-06,
+      "loss": 1.3601,
+      "step": 1662
+    },
+    {
+      "epoch": 2.0696950840074675,
+      "grad_norm": 3.3320810794830322,
+      "learning_rate": 1.7573800738007383e-06,
+      "loss": 1.3454,
+      "step": 1663
+    },
+    {
+      "epoch": 2.0709396390790293,
+      "grad_norm": 3.475529909133911,
+      "learning_rate": 1.7550738007380075e-06,
+      "loss": 1.3666,
+      "step": 1664
+    },
+    {
+      "epoch": 2.072184194150591,
+      "grad_norm": 3.302342653274536,
+      "learning_rate": 1.752767527675277e-06,
+      "loss": 1.3824,
+      "step": 1665
+    },
+    {
+      "epoch": 2.073428749222153,
+      "grad_norm": 3.198641777038574,
+      "learning_rate": 1.7504612546125463e-06,
+      "loss": 1.3398,
+      "step": 1666
+    },
+    {
+      "epoch": 2.074673304293715,
+      "grad_norm": 3.421834945678711,
+      "learning_rate": 1.7481549815498156e-06,
+      "loss": 1.363,
+      "step": 1667
+    },
+    {
+      "epoch": 2.0759178593652767,
+      "grad_norm": 3.654343843460083,
+      "learning_rate": 1.7458487084870852e-06,
+      "loss": 1.3852,
+      "step": 1668
+    },
+    {
+      "epoch": 2.077162414436839,
+      "grad_norm": 3.3543035984039307,
+      "learning_rate": 1.7435424354243544e-06,
+      "loss": 1.3554,
+      "step": 1669
+    },
+    {
+      "epoch": 2.078406969508401,
+      "grad_norm": 3.2144811153411865,
+      "learning_rate": 1.7412361623616236e-06,
+      "loss": 1.3137,
+      "step": 1670
+    },
+    {
+      "epoch": 2.078406969508401,
+      "eval_loss": 1.6155465841293335,
+      "eval_runtime": 52.4111,
+      "eval_samples_per_second": 19.08,
+      "eval_steps_per_second": 0.801,
+      "step": 1670
+    },
+    {
+      "epoch": 2.0796515245799627,
+      "grad_norm": 3.2240991592407227,
+      "learning_rate": 1.7389298892988932e-06,
+      "loss": 1.3578,
+      "step": 1671
+    },
+    {
+      "epoch": 2.0808960796515246,
+      "grad_norm": 3.1450018882751465,
+      "learning_rate": 1.7366236162361625e-06,
+      "loss": 1.2999,
+      "step": 1672
+    },
+    {
+      "epoch": 2.0821406347230864,
+      "grad_norm": 3.4396843910217285,
+      "learning_rate": 1.7343173431734319e-06,
+      "loss": 1.3522,
+      "step": 1673
+    },
+    {
+      "epoch": 2.0833851897946483,
+      "grad_norm": 3.1719677448272705,
+      "learning_rate": 1.7320110701107013e-06,
+      "loss": 1.3552,
+      "step": 1674
+    },
+    {
+      "epoch": 2.08462974486621,
+      "grad_norm": 3.4420199394226074,
+      "learning_rate": 1.7297047970479705e-06,
+      "loss": 1.4033,
+      "step": 1675
+    },
+    {
+      "epoch": 2.0858742999377724,
+      "grad_norm": 3.3366570472717285,
+      "learning_rate": 1.7273985239852401e-06,
+      "loss": 1.3407,
+      "step": 1676
+    },
+    {
+      "epoch": 2.0871188550093343,
+      "grad_norm": 3.3416335582733154,
+      "learning_rate": 1.7250922509225094e-06,
+      "loss": 1.3882,
+      "step": 1677
+    },
+    {
+      "epoch": 2.088363410080896,
+      "grad_norm": 3.4189534187316895,
+      "learning_rate": 1.7227859778597788e-06,
+      "loss": 1.3996,
+      "step": 1678
+    },
+    {
+      "epoch": 2.089607965152458,
+      "grad_norm": 3.309095859527588,
+      "learning_rate": 1.7204797047970482e-06,
+      "loss": 1.3799,
+      "step": 1679
+    },
+    {
+      "epoch": 2.09085252022402,
+      "grad_norm": 3.369511365890503,
+      "learning_rate": 1.7181734317343174e-06,
+      "loss": 1.4159,
+      "step": 1680
+    },
+    {
+      "epoch": 2.09085252022402,
+      "eval_loss": 1.6145448684692383,
+      "eval_runtime": 49.0228,
+      "eval_samples_per_second": 20.399,
+      "eval_steps_per_second": 0.857,
+      "step": 1680
+    },
+    {
+      "epoch": 2.0920970752955816,
+      "grad_norm": 3.323286771774292,
+      "learning_rate": 1.7158671586715868e-06,
+      "loss": 1.4035,
+      "step": 1681
+    },
+    {
+      "epoch": 2.093341630367144,
+      "grad_norm": 3.3053157329559326,
+      "learning_rate": 1.7135608856088563e-06,
+      "loss": 1.3498,
+      "step": 1682
+    },
+    {
+      "epoch": 2.094586185438706,
+      "grad_norm": 3.141890048980713,
+      "learning_rate": 1.7112546125461257e-06,
+      "loss": 1.3438,
+      "step": 1683
+    },
+    {
+      "epoch": 2.0958307405102676,
+      "grad_norm": 3.135058879852295,
+      "learning_rate": 1.708948339483395e-06,
+      "loss": 1.3726,
+      "step": 1684
+    },
+    {
+      "epoch": 2.0970752955818295,
+      "grad_norm": 3.2389063835144043,
+      "learning_rate": 1.7066420664206643e-06,
+      "loss": 1.3661,
+      "step": 1685
+    },
+    {
+      "epoch": 2.0983198506533913,
+      "grad_norm": 3.1741721630096436,
+      "learning_rate": 1.7043357933579337e-06,
+      "loss": 1.3438,
+      "step": 1686
+    },
+    {
+      "epoch": 2.099564405724953,
+      "grad_norm": 3.3524587154388428,
+      "learning_rate": 1.7020295202952032e-06,
+      "loss": 1.3388,
+      "step": 1687
+    },
+    {
+      "epoch": 2.100808960796515,
+      "grad_norm": 3.3574740886688232,
+      "learning_rate": 1.6997232472324724e-06,
+      "loss": 1.3702,
+      "step": 1688
+    },
+    {
+      "epoch": 2.1020535158680773,
+      "grad_norm": 3.458645820617676,
+      "learning_rate": 1.6974169741697418e-06,
+      "loss": 1.3785,
+      "step": 1689
+    },
+    {
+      "epoch": 2.103298070939639,
+      "grad_norm": 3.346677303314209,
+      "learning_rate": 1.6951107011070112e-06,
+      "loss": 1.3295,
+      "step": 1690
+    },
+    {
+      "epoch": 2.103298070939639,
+      "eval_loss": 1.6184555292129517,
+      "eval_runtime": 46.6544,
+      "eval_samples_per_second": 21.434,
+      "eval_steps_per_second": 0.9,
+      "step": 1690
+    },
+    {
+      "epoch": 2.104542626011201,
+      "grad_norm": 3.3403804302215576,
+      "learning_rate": 1.6928044280442806e-06,
+      "loss": 1.3912,
+      "step": 1691
+    },
+    {
+      "epoch": 2.105787181082763,
+      "grad_norm": 3.2638344764709473,
+      "learning_rate": 1.6904981549815499e-06,
+      "loss": 1.3684,
+      "step": 1692
+    },
+    {
+      "epoch": 2.1070317361543247,
+      "grad_norm": 3.192701578140259,
+      "learning_rate": 1.6881918819188193e-06,
+      "loss": 1.2997,
+      "step": 1693
+    },
+    {
+      "epoch": 2.1082762912258866,
+      "grad_norm": 3.482121706008911,
+      "learning_rate": 1.6858856088560887e-06,
+      "loss": 1.3515,
+      "step": 1694
+    },
+    {
+      "epoch": 2.109520846297449,
+      "grad_norm": 3.292353630065918,
+      "learning_rate": 1.6835793357933581e-06,
+      "loss": 1.3603,
+      "step": 1695
+    },
+    {
+      "epoch": 2.1107654013690107,
+      "grad_norm": 3.34220290184021,
+      "learning_rate": 1.6812730627306275e-06,
+      "loss": 1.3666,
+      "step": 1696
+    },
+    {
+      "epoch": 2.1120099564405725,
+      "grad_norm": 3.190058469772339,
+      "learning_rate": 1.6789667896678968e-06,
+      "loss": 1.3549,
+      "step": 1697
+    },
+    {
+      "epoch": 2.1132545115121344,
+      "grad_norm": 3.3759775161743164,
+      "learning_rate": 1.6766605166051662e-06,
+      "loss": 1.3628,
+      "step": 1698
+    },
+    {
+      "epoch": 2.1144990665836962,
+      "grad_norm": 3.419440507888794,
+      "learning_rate": 1.6743542435424356e-06,
+      "loss": 1.3618,
+      "step": 1699
+    },
+    {
+      "epoch": 2.115743621655258,
+      "grad_norm": 3.5021324157714844,
+      "learning_rate": 1.6720479704797048e-06,
+      "loss": 1.4023,
+      "step": 1700
+    },
+    {
+      "epoch": 2.115743621655258,
+      "eval_loss": 1.6218053102493286,
+      "eval_runtime": 48.3483,
+      "eval_samples_per_second": 20.683,
+      "eval_steps_per_second": 0.869,
+      "step": 1700
+    },
+    {
+      "epoch": 2.11698817672682,
+      "grad_norm": 3.449636936187744,
+      "learning_rate": 1.6697416974169742e-06,
+      "loss": 1.3534,
+      "step": 1701
+    },
+    {
+      "epoch": 2.1182327317983822,
+      "grad_norm": 3.483173370361328,
+      "learning_rate": 1.6674354243542437e-06,
+      "loss": 1.4053,
+      "step": 1702
+    },
+    {
+      "epoch": 2.119477286869944,
+      "grad_norm": 3.3713502883911133,
+      "learning_rate": 1.665129151291513e-06,
+      "loss": 1.3352,
+      "step": 1703
+    },
+    {
+      "epoch": 2.120721841941506,
+      "grad_norm": 3.2565910816192627,
+      "learning_rate": 1.6628228782287825e-06,
+      "loss": 1.3751,
+      "step": 1704
+    },
+    {
+      "epoch": 2.1219663970130678,
+      "grad_norm": 3.1434268951416016,
+      "learning_rate": 1.6605166051660517e-06,
+      "loss": 1.346,
+      "step": 1705
+    },
+    {
+      "epoch": 2.1232109520846296,
+      "grad_norm": 3.229909896850586,
+      "learning_rate": 1.6582103321033211e-06,
+      "loss": 1.3477,
+      "step": 1706
+    },
+    {
+      "epoch": 2.1244555071561915,
+      "grad_norm": 3.3930044174194336,
+      "learning_rate": 1.6559040590405906e-06,
+      "loss": 1.3374,
+      "step": 1707
+    },
+    {
+      "epoch": 2.1257000622277538,
+      "grad_norm": 3.2674202919006348,
+      "learning_rate": 1.6535977859778598e-06,
+      "loss": 1.321,
+      "step": 1708
+    },
+    {
+      "epoch": 2.1269446172993156,
+      "grad_norm": 3.3048641681671143,
+      "learning_rate": 1.6512915129151294e-06,
+      "loss": 1.3641,
+      "step": 1709
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "grad_norm": 3.4714198112487793,
+      "learning_rate": 1.6489852398523986e-06,
+      "loss": 1.3931,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "eval_loss": 1.6198205947875977,
+      "eval_runtime": 42.9272,
+      "eval_samples_per_second": 23.295,
+      "eval_steps_per_second": 0.978,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1294337274424393,
+      "grad_norm": 3.440058946609497,
+      "learning_rate": 1.6466789667896678e-06,
+      "loss": 1.3892,
+      "step": 1711
+    },
+    {
+      "epoch": 2.130678282514001,
+      "grad_norm": 3.4067635536193848,
+      "learning_rate": 1.6443726937269375e-06,
+      "loss": 1.3719,
+      "step": 1712
+    },
+    {
+      "epoch": 2.131922837585563,
+      "grad_norm": 3.285536766052246,
+      "learning_rate": 1.6420664206642067e-06,
+      "loss": 1.3735,
+      "step": 1713
+    },
+    {
+      "epoch": 2.133167392657125,
+      "grad_norm": 3.171344041824341,
+      "learning_rate": 1.6397601476014763e-06,
+      "loss": 1.3628,
+      "step": 1714
+    },
+    {
+      "epoch": 2.134411947728687,
+      "grad_norm": 3.3051321506500244,
+      "learning_rate": 1.6374538745387455e-06,
+      "loss": 1.3579,
+      "step": 1715
+    },
+    {
+      "epoch": 2.135656502800249,
+      "grad_norm": 3.2083475589752197,
+      "learning_rate": 1.6351476014760147e-06,
+      "loss": 1.297,
+      "step": 1716
+    },
+    {
+      "epoch": 2.136901057871811,
+      "grad_norm": 3.340172052383423,
+      "learning_rate": 1.6328413284132844e-06,
+      "loss": 1.3468,
+      "step": 1717
+    },
+    {
+      "epoch": 2.1381456129433727,
+      "grad_norm": 3.3475770950317383,
+      "learning_rate": 1.6305350553505536e-06,
+      "loss": 1.3918,
+      "step": 1718
+    },
+    {
+      "epoch": 2.1393901680149345,
+      "grad_norm": 3.4087703227996826,
+      "learning_rate": 1.6282287822878228e-06,
+      "loss": 1.4083,
+      "step": 1719
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "grad_norm": 3.461160898208618,
+      "learning_rate": 1.6259225092250924e-06,
+      "loss": 1.3471,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "eval_loss": 1.6216135025024414,
+      "eval_runtime": 49.1097,
+      "eval_samples_per_second": 20.363,
+      "eval_steps_per_second": 0.855,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1418792781580587,
+      "grad_norm": 3.3619799613952637,
+      "learning_rate": 1.6236162361623616e-06,
+      "loss": 1.3235,
+      "step": 1721
+    },
+    {
+      "epoch": 2.1431238332296205,
+      "grad_norm": 3.332010507583618,
+      "learning_rate": 1.6213099630996313e-06,
+      "loss": 1.3576,
+      "step": 1722
+    },
+    {
+      "epoch": 2.1443683883011824,
+      "grad_norm": 3.300055503845215,
+      "learning_rate": 1.6190036900369005e-06,
+      "loss": 1.3269,
+      "step": 1723
+    },
+    {
+      "epoch": 2.145612943372744,
+      "grad_norm": 3.3326926231384277,
+      "learning_rate": 1.6166974169741697e-06,
+      "loss": 1.346,
+      "step": 1724
+    },
+    {
+      "epoch": 2.146857498444306,
+      "grad_norm": 3.4713706970214844,
+      "learning_rate": 1.6143911439114393e-06,
+      "loss": 1.3416,
+      "step": 1725
+    },
+    {
+      "epoch": 2.148102053515868,
+      "grad_norm": 3.1899967193603516,
+      "learning_rate": 1.6120848708487085e-06,
+      "loss": 1.3192,
+      "step": 1726
+    },
+    {
+      "epoch": 2.1493466085874298,
+      "grad_norm": 3.1781857013702393,
+      "learning_rate": 1.6097785977859782e-06,
+      "loss": 1.3197,
+      "step": 1727
+    },
+    {
+      "epoch": 2.150591163658992,
+      "grad_norm": 3.3369524478912354,
+      "learning_rate": 1.6074723247232474e-06,
+      "loss": 1.3804,
+      "step": 1728
+    },
+    {
+      "epoch": 2.151835718730554,
+      "grad_norm": 3.2813405990600586,
+      "learning_rate": 1.6051660516605166e-06,
+      "loss": 1.3778,
+      "step": 1729
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "grad_norm": 3.3395135402679443,
+      "learning_rate": 1.6028597785977862e-06,
+      "loss": 1.3523,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "eval_loss": 1.6181366443634033,
+      "eval_runtime": 50.7133,
+      "eval_samples_per_second": 19.719,
+      "eval_steps_per_second": 0.828,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1543248288736776,
+      "grad_norm": 3.2905092239379883,
+      "learning_rate": 1.6005535055350554e-06,
+      "loss": 1.3568,
+      "step": 1731
+    },
+    {
+      "epoch": 2.1555693839452394,
+      "grad_norm": 3.314086437225342,
+      "learning_rate": 1.5982472324723247e-06,
+      "loss": 1.3543,
+      "step": 1732
+    },
+    {
+      "epoch": 2.1568139390168013,
+      "grad_norm": 3.340034008026123,
+      "learning_rate": 1.5959409594095943e-06,
+      "loss": 1.3639,
+      "step": 1733
+    },
+    {
+      "epoch": 2.1580584940883636,
+      "grad_norm": 3.264814615249634,
+      "learning_rate": 1.5936346863468635e-06,
+      "loss": 1.3599,
+      "step": 1734
+    },
+    {
+      "epoch": 2.1593030491599254,
+      "grad_norm": 3.2429778575897217,
+      "learning_rate": 1.5913284132841331e-06,
+      "loss": 1.3379,
+      "step": 1735
+    },
+    {
+      "epoch": 2.1605476042314873,
+      "grad_norm": 3.221935987472534,
+      "learning_rate": 1.5890221402214023e-06,
+      "loss": 1.3162,
+      "step": 1736
+    },
+    {
+      "epoch": 2.161792159303049,
+      "grad_norm": 3.266073703765869,
+      "learning_rate": 1.5867158671586716e-06,
+      "loss": 1.3632,
+      "step": 1737
+    },
+    {
+      "epoch": 2.163036714374611,
+      "grad_norm": 3.2618112564086914,
+      "learning_rate": 1.5844095940959412e-06,
+      "loss": 1.3779,
+      "step": 1738
+    },
+    {
+      "epoch": 2.164281269446173,
+      "grad_norm": 3.2474825382232666,
+      "learning_rate": 1.5821033210332104e-06,
+      "loss": 1.3201,
+      "step": 1739
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "grad_norm": 3.4494247436523438,
+      "learning_rate": 1.57979704797048e-06,
+      "loss": 1.3949,
+      "step": 1740
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "eval_loss": 1.6180604696273804,
+      "eval_runtime": 49.3152,
+      "eval_samples_per_second": 20.278,
+      "eval_steps_per_second": 0.852,
+      "step": 1740
+    },
+    {
+      "epoch": 2.166770379589297,
+      "grad_norm": 3.221304416656494,
+      "learning_rate": 1.5774907749077492e-06,
+      "loss": 1.3341,
+      "step": 1741
+    },
+    {
+      "epoch": 2.168014934660859,
+      "grad_norm": 3.28930401802063,
+      "learning_rate": 1.5751845018450185e-06,
+      "loss": 1.3151,
+      "step": 1742
+    },
+    {
+      "epoch": 2.1692594897324207,
+      "grad_norm": 3.3582026958465576,
+      "learning_rate": 1.572878228782288e-06,
+      "loss": 1.3767,
+      "step": 1743
+    },
+    {
+      "epoch": 2.1705040448039825,
+      "grad_norm": 3.2603683471679688,
+      "learning_rate": 1.5705719557195573e-06,
+      "loss": 1.3644,
+      "step": 1744
+    },
+    {
+      "epoch": 2.1717485998755444,
+      "grad_norm": 3.3291070461273193,
+      "learning_rate": 1.5682656826568267e-06,
+      "loss": 1.3226,
+      "step": 1745
+    },
+    {
+      "epoch": 2.172993154947106,
+      "grad_norm": 3.473127841949463,
+      "learning_rate": 1.5659594095940962e-06,
+      "loss": 1.3531,
+      "step": 1746
+    },
+    {
+      "epoch": 2.1742377100186685,
+      "grad_norm": 3.2014408111572266,
+      "learning_rate": 1.5636531365313654e-06,
+      "loss": 1.3634,
+      "step": 1747
+    },
+    {
+      "epoch": 2.1754822650902303,
+      "grad_norm": 3.56447434425354,
+      "learning_rate": 1.561346863468635e-06,
+      "loss": 1.3788,
+      "step": 1748
+    },
+    {
+      "epoch": 2.176726820161792,
+      "grad_norm": 3.351318359375,
+      "learning_rate": 1.5590405904059042e-06,
+      "loss": 1.3133,
+      "step": 1749
+    },
+    {
+      "epoch": 2.177971375233354,
+      "grad_norm": 3.278183937072754,
+      "learning_rate": 1.5567343173431734e-06,
+      "loss": 1.3164,
+      "step": 1750
+    },
+    {
+      "epoch": 2.177971375233354,
+      "eval_loss": 1.6197904348373413,
+      "eval_runtime": 50.2776,
+      "eval_samples_per_second": 19.89,
+      "eval_steps_per_second": 0.835,
+      "step": 1750
+    },
+    {
+      "epoch": 2.179215930304916,
+      "grad_norm": 3.379854917526245,
+      "learning_rate": 1.554428044280443e-06,
+      "loss": 1.3785,
+      "step": 1751
+    },
+    {
+      "epoch": 2.1804604853764777,
+      "grad_norm": 3.2265610694885254,
+      "learning_rate": 1.5521217712177123e-06,
+      "loss": 1.3483,
+      "step": 1752
+    },
+    {
+      "epoch": 2.18170504044804,
+      "grad_norm": 3.2259609699249268,
+      "learning_rate": 1.5498154981549817e-06,
+      "loss": 1.377,
+      "step": 1753
+    },
+    {
+      "epoch": 2.182949595519602,
+      "grad_norm": 3.1671273708343506,
+      "learning_rate": 1.5475092250922511e-06,
+      "loss": 1.3106,
+      "step": 1754
+    },
+    {
+      "epoch": 2.1841941505911637,
+      "grad_norm": 3.280299425125122,
+      "learning_rate": 1.5452029520295203e-06,
+      "loss": 1.3271,
+      "step": 1755
+    },
+    {
+      "epoch": 2.1854387056627256,
+      "grad_norm": 3.224238634109497,
+      "learning_rate": 1.5428966789667897e-06,
+      "loss": 1.3369,
+      "step": 1756
+    },
+    {
+      "epoch": 2.1866832607342874,
+      "grad_norm": 3.3139021396636963,
+      "learning_rate": 1.5405904059040592e-06,
+      "loss": 1.3562,
+      "step": 1757
+    },
+    {
+      "epoch": 2.1879278158058493,
+      "grad_norm": 3.340569257736206,
+      "learning_rate": 1.5382841328413286e-06,
+      "loss": 1.3526,
+      "step": 1758
+    },
+    {
+      "epoch": 2.189172370877411,
+      "grad_norm": 3.505159854888916,
+      "learning_rate": 1.535977859778598e-06,
+      "loss": 1.3796,
+      "step": 1759
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "grad_norm": 3.351260185241699,
+      "learning_rate": 1.5336715867158672e-06,
+      "loss": 1.3676,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "eval_loss": 1.6169729232788086,
+      "eval_runtime": 48.0801,
+      "eval_samples_per_second": 20.799,
+      "eval_steps_per_second": 0.874,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1916614810205353,
+      "grad_norm": 3.4291462898254395,
+      "learning_rate": 1.5313653136531366e-06,
+      "loss": 1.3574,
+      "step": 1761
+    },
+    {
+      "epoch": 2.192906036092097,
+      "grad_norm": 3.5459227561950684,
+      "learning_rate": 1.529059040590406e-06,
+      "loss": 1.3919,
+      "step": 1762
+    },
+    {
+      "epoch": 2.194150591163659,
+      "grad_norm": 3.298430919647217,
+      "learning_rate": 1.5267527675276755e-06,
+      "loss": 1.3551,
+      "step": 1763
+    },
+    {
+      "epoch": 2.195395146235221,
+      "grad_norm": 3.4521286487579346,
+      "learning_rate": 1.5244464944649447e-06,
+      "loss": 1.3779,
+      "step": 1764
+    },
+    {
+      "epoch": 2.1966397013067827,
+      "grad_norm": 3.354443073272705,
+      "learning_rate": 1.5221402214022141e-06,
+      "loss": 1.3574,
+      "step": 1765
+    },
+    {
+      "epoch": 2.197884256378345,
+      "grad_norm": 3.069854259490967,
+      "learning_rate": 1.5198339483394836e-06,
+      "loss": 1.2987,
+      "step": 1766
+    },
+    {
+      "epoch": 2.199128811449907,
+      "grad_norm": 3.0894274711608887,
+      "learning_rate": 1.517527675276753e-06,
+      "loss": 1.3408,
+      "step": 1767
+    },
+    {
+      "epoch": 2.2003733665214686,
+      "grad_norm": 3.3033676147460938,
+      "learning_rate": 1.5152214022140222e-06,
+      "loss": 1.3394,
+      "step": 1768
+    },
+    {
+      "epoch": 2.2016179215930305,
+      "grad_norm": 3.2894108295440674,
+      "learning_rate": 1.5129151291512916e-06,
+      "loss": 1.3505,
+      "step": 1769
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "grad_norm": 3.263564109802246,
+      "learning_rate": 1.510608856088561e-06,
+      "loss": 1.3318,
+      "step": 1770
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "eval_loss": 1.6183674335479736,
+      "eval_runtime": 48.3189,
+      "eval_samples_per_second": 20.696,
+      "eval_steps_per_second": 0.869,
+      "step": 1770
+    },
+    {
+      "epoch": 2.204107031736154,
+      "grad_norm": 3.2933449745178223,
+      "learning_rate": 1.5083025830258305e-06,
+      "loss": 1.3397,
+      "step": 1771
+    },
+    {
+      "epoch": 2.205351586807716,
+      "grad_norm": 3.3185815811157227,
+      "learning_rate": 1.5059963099630997e-06,
+      "loss": 1.3339,
+      "step": 1772
+    },
+    {
+      "epoch": 2.2065961418792783,
+      "grad_norm": 3.445251703262329,
+      "learning_rate": 1.503690036900369e-06,
+      "loss": 1.3898,
+      "step": 1773
+    },
+    {
+      "epoch": 2.20784069695084,
+      "grad_norm": 3.399627685546875,
+      "learning_rate": 1.5013837638376385e-06,
+      "loss": 1.3646,
+      "step": 1774
+    },
+    {
+      "epoch": 2.209085252022402,
+      "grad_norm": 3.3483574390411377,
+      "learning_rate": 1.499077490774908e-06,
+      "loss": 1.3732,
+      "step": 1775
+    },
+    {
+      "epoch": 2.210329807093964,
+      "grad_norm": 3.3097689151763916,
+      "learning_rate": 1.4967712177121774e-06,
+      "loss": 1.3317,
+      "step": 1776
+    },
+    {
+      "epoch": 2.2115743621655257,
+      "grad_norm": 3.254025459289551,
+      "learning_rate": 1.4944649446494466e-06,
+      "loss": 1.3818,
+      "step": 1777
+    },
+    {
+      "epoch": 2.2128189172370876,
+      "grad_norm": 3.2063064575195312,
+      "learning_rate": 1.492158671586716e-06,
+      "loss": 1.3357,
+      "step": 1778
+    },
+    {
+      "epoch": 2.21406347230865,
+      "grad_norm": 3.1974005699157715,
+      "learning_rate": 1.4898523985239854e-06,
+      "loss": 1.4103,
+      "step": 1779
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "grad_norm": 3.1551153659820557,
+      "learning_rate": 1.4875461254612546e-06,
+      "loss": 1.3088,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "eval_loss": 1.6195781230926514,
+      "eval_runtime": 52.379,
+      "eval_samples_per_second": 19.092,
+      "eval_steps_per_second": 0.802,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2165525824517736,
+      "grad_norm": 3.3186404705047607,
+      "learning_rate": 1.485239852398524e-06,
+      "loss": 1.3775,
+      "step": 1781
+    },
+    {
+      "epoch": 2.2177971375233354,
+      "grad_norm": 3.2764358520507812,
+      "learning_rate": 1.4829335793357935e-06,
+      "loss": 1.3253,
+      "step": 1782
+    },
+    {
+      "epoch": 2.2190416925948973,
+      "grad_norm": 3.3535802364349365,
+      "learning_rate": 1.4806273062730627e-06,
+      "loss": 1.3451,
+      "step": 1783
+    },
+    {
+      "epoch": 2.220286247666459,
+      "grad_norm": 3.304255962371826,
+      "learning_rate": 1.4783210332103323e-06,
+      "loss": 1.3565,
+      "step": 1784
+    },
+    {
+      "epoch": 2.2215308027380214,
+      "grad_norm": 3.281244993209839,
+      "learning_rate": 1.4760147601476015e-06,
+      "loss": 1.3458,
+      "step": 1785
+    },
+    {
+      "epoch": 2.2227753578095832,
+      "grad_norm": 3.4199814796447754,
+      "learning_rate": 1.473708487084871e-06,
+      "loss": 1.359,
+      "step": 1786
+    },
+    {
+      "epoch": 2.224019912881145,
+      "grad_norm": 3.298577070236206,
+      "learning_rate": 1.4714022140221404e-06,
+      "loss": 1.3298,
+      "step": 1787
+    },
+    {
+      "epoch": 2.225264467952707,
+      "grad_norm": 3.2206099033355713,
+      "learning_rate": 1.4690959409594096e-06,
+      "loss": 1.3026,
+      "step": 1788
+    },
+    {
+      "epoch": 2.226509023024269,
+      "grad_norm": 3.3583920001983643,
+      "learning_rate": 1.4667896678966792e-06,
+      "loss": 1.3618,
+      "step": 1789
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "grad_norm": 3.4446728229522705,
+      "learning_rate": 1.4644833948339484e-06,
+      "loss": 1.3895,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "eval_loss": 1.6163197755813599,
+      "eval_runtime": 49.4171,
+      "eval_samples_per_second": 20.236,
+      "eval_steps_per_second": 0.85,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2289981331673925,
+      "grad_norm": 3.329155445098877,
+      "learning_rate": 1.4621771217712176e-06,
+      "loss": 1.3368,
+      "step": 1791
+    },
+    {
+      "epoch": 2.2302426882389548,
+      "grad_norm": 3.4599087238311768,
+      "learning_rate": 1.4598708487084873e-06,
+      "loss": 1.3487,
+      "step": 1792
+    },
+    {
+      "epoch": 2.2314872433105166,
+      "grad_norm": 3.3162965774536133,
+      "learning_rate": 1.4575645756457565e-06,
+      "loss": 1.3493,
+      "step": 1793
+    },
+    {
+      "epoch": 2.2327317983820785,
+      "grad_norm": 3.1206021308898926,
+      "learning_rate": 1.4552583025830261e-06,
+      "loss": 1.3506,
+      "step": 1794
+    },
+    {
+      "epoch": 2.2339763534536403,
+      "grad_norm": 3.334165573120117,
+      "learning_rate": 1.4529520295202953e-06,
+      "loss": 1.3315,
+      "step": 1795
+    },
+    {
+      "epoch": 2.235220908525202,
+      "grad_norm": 3.3404204845428467,
+      "learning_rate": 1.4506457564575645e-06,
+      "loss": 1.3163,
+      "step": 1796
+    },
+    {
+      "epoch": 2.236465463596764,
+      "grad_norm": 3.2753002643585205,
+      "learning_rate": 1.4483394833948342e-06,
+      "loss": 1.3567,
+      "step": 1797
+    },
+    {
+      "epoch": 2.2377100186683263,
+      "grad_norm": 3.42242431640625,
+      "learning_rate": 1.4460332103321034e-06,
+      "loss": 1.3761,
+      "step": 1798
+    },
+    {
+      "epoch": 2.238954573739888,
+      "grad_norm": 3.31380033493042,
+      "learning_rate": 1.4437269372693726e-06,
+      "loss": 1.3353,
+      "step": 1799
+    },
+    {
+      "epoch": 2.24019912881145,
+      "grad_norm": 3.239811658859253,
+      "learning_rate": 1.4414206642066422e-06,
+      "loss": 1.3187,
+      "step": 1800
+    },
+    {
+      "epoch": 2.24019912881145,
+      "eval_loss": 1.6201411485671997,
+      "eval_runtime": 53.5369,
+      "eval_samples_per_second": 18.679,
+      "eval_steps_per_second": 0.785,
+      "step": 1800
+    },
+    {
+      "epoch": 2.241443683883012,
+      "grad_norm": 3.549010753631592,
+      "learning_rate": 1.4391143911439114e-06,
+      "loss": 1.3633,
+      "step": 1801
+    },
+    {
+      "epoch": 2.2426882389545737,
+      "grad_norm": 3.1888253688812256,
+      "learning_rate": 1.436808118081181e-06,
+      "loss": 1.3292,
+      "step": 1802
+    },
+    {
+      "epoch": 2.2439327940261355,
+      "grad_norm": 3.1015641689300537,
+      "learning_rate": 1.4345018450184503e-06,
+      "loss": 1.3404,
+      "step": 1803
+    },
+    {
+      "epoch": 2.2451773490976974,
+      "grad_norm": 3.2635490894317627,
+      "learning_rate": 1.4321955719557195e-06,
+      "loss": 1.3181,
+      "step": 1804
+    },
+    {
+      "epoch": 2.2464219041692597,
+      "grad_norm": 3.314697265625,
+      "learning_rate": 1.4298892988929891e-06,
+      "loss": 1.308,
+      "step": 1805
+    },
+    {
+      "epoch": 2.2476664592408215,
+      "grad_norm": 3.3572986125946045,
+      "learning_rate": 1.4275830258302584e-06,
+      "loss": 1.2888,
+      "step": 1806
+    },
+    {
+      "epoch": 2.2489110143123834,
+      "grad_norm": 3.403423547744751,
+      "learning_rate": 1.425276752767528e-06,
+      "loss": 1.3438,
+      "step": 1807
+    },
+    {
+      "epoch": 2.2501555693839452,
+      "grad_norm": 3.5180821418762207,
+      "learning_rate": 1.4229704797047972e-06,
+      "loss": 1.3613,
+      "step": 1808
+    },
+    {
+      "epoch": 2.251400124455507,
+      "grad_norm": 3.5315089225769043,
+      "learning_rate": 1.4206642066420664e-06,
+      "loss": 1.3655,
+      "step": 1809
+    },
+    {
+      "epoch": 2.252644679527069,
+      "grad_norm": 3.4266903400421143,
+      "learning_rate": 1.418357933579336e-06,
+      "loss": 1.3481,
+      "step": 1810
+    },
+    {
+      "epoch": 2.252644679527069,
+      "eval_loss": 1.619051456451416,
+      "eval_runtime": 43.1595,
+      "eval_samples_per_second": 23.17,
+      "eval_steps_per_second": 0.973,
+      "step": 1810
+    },
+    {
+      "epoch": 2.253889234598631,
+      "grad_norm": 3.412053108215332,
+      "learning_rate": 1.4160516605166053e-06,
+      "loss": 1.3097,
+      "step": 1811
+    },
+    {
+      "epoch": 2.255133789670193,
+      "grad_norm": 3.4371933937072754,
+      "learning_rate": 1.4137453874538745e-06,
+      "loss": 1.3831,
+      "step": 1812
+    },
+    {
+      "epoch": 2.256378344741755,
+      "grad_norm": 3.331648588180542,
+      "learning_rate": 1.411439114391144e-06,
+      "loss": 1.3701,
+      "step": 1813
+    },
+    {
+      "epoch": 2.2576228998133168,
+      "grad_norm": 3.3840672969818115,
+      "learning_rate": 1.4091328413284133e-06,
+      "loss": 1.3676,
+      "step": 1814
+    },
+    {
+      "epoch": 2.2588674548848786,
+      "grad_norm": 3.3289988040924072,
+      "learning_rate": 1.406826568265683e-06,
+      "loss": 1.3281,
+      "step": 1815
+    },
+    {
+      "epoch": 2.2601120099564405,
+      "grad_norm": 3.309511423110962,
+      "learning_rate": 1.4045202952029522e-06,
+      "loss": 1.3216,
+      "step": 1816
+    },
+    {
+      "epoch": 2.2613565650280023,
+      "grad_norm": 3.3434019088745117,
+      "learning_rate": 1.4022140221402214e-06,
+      "loss": 1.3261,
+      "step": 1817
+    },
+    {
+      "epoch": 2.2626011200995646,
+      "grad_norm": 3.3371620178222656,
+      "learning_rate": 1.399907749077491e-06,
+      "loss": 1.3597,
+      "step": 1818
+    },
+    {
+      "epoch": 2.2638456751711264,
+      "grad_norm": 3.287353515625,
+      "learning_rate": 1.3976014760147602e-06,
+      "loss": 1.3431,
+      "step": 1819
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "grad_norm": 3.422102689743042,
+      "learning_rate": 1.3952952029520298e-06,
+      "loss": 1.3454,
+      "step": 1820
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "eval_loss": 1.6200023889541626,
+      "eval_runtime": 46.3983,
+      "eval_samples_per_second": 21.552,
+      "eval_steps_per_second": 0.905,
+      "step": 1820
+    },
+    {
+      "epoch": 2.26633478531425,
+      "grad_norm": 3.6252498626708984,
+      "learning_rate": 1.392988929889299e-06,
+      "loss": 1.4066,
+      "step": 1821
+    },
+    {
+      "epoch": 2.267579340385812,
+      "grad_norm": 3.423708915710449,
+      "learning_rate": 1.3906826568265683e-06,
+      "loss": 1.3504,
+      "step": 1822
+    },
+    {
+      "epoch": 2.268823895457374,
+      "grad_norm": 3.3063361644744873,
+      "learning_rate": 1.388376383763838e-06,
+      "loss": 1.3635,
+      "step": 1823
+    },
+    {
+      "epoch": 2.270068450528936,
+      "grad_norm": 3.380657911300659,
+      "learning_rate": 1.3860701107011071e-06,
+      "loss": 1.3442,
+      "step": 1824
+    },
+    {
+      "epoch": 2.271313005600498,
+      "grad_norm": 3.300102472305298,
+      "learning_rate": 1.3837638376383765e-06,
+      "loss": 1.3741,
+      "step": 1825
+    },
+    {
+      "epoch": 2.27255756067206,
+      "grad_norm": 3.3895652294158936,
+      "learning_rate": 1.381457564575646e-06,
+      "loss": 1.3288,
+      "step": 1826
+    },
+    {
+      "epoch": 2.2738021157436217,
+      "grad_norm": 3.4393489360809326,
+      "learning_rate": 1.3791512915129152e-06,
+      "loss": 1.3544,
+      "step": 1827
+    },
+    {
+      "epoch": 2.2750466708151835,
+      "grad_norm": 3.4510536193847656,
+      "learning_rate": 1.3768450184501846e-06,
+      "loss": 1.3894,
+      "step": 1828
+    },
+    {
+      "epoch": 2.2762912258867454,
+      "grad_norm": 3.39282488822937,
+      "learning_rate": 1.374538745387454e-06,
+      "loss": 1.3776,
+      "step": 1829
+    },
+    {
+      "epoch": 2.277535780958307,
+      "grad_norm": 3.250432014465332,
+      "learning_rate": 1.3722324723247232e-06,
+      "loss": 1.3511,
+      "step": 1830
+    },
+    {
+      "epoch": 2.277535780958307,
+      "eval_loss": 1.6178572177886963,
+      "eval_runtime": 45.3086,
+      "eval_samples_per_second": 22.071,
+      "eval_steps_per_second": 0.927,
+      "step": 1830
+    },
+    {
+      "epoch": 2.2787803360298695,
+      "grad_norm": 3.4044647216796875,
+      "learning_rate": 1.3699261992619929e-06,
+      "loss": 1.3548,
+      "step": 1831
+    },
+    {
+      "epoch": 2.2800248911014314,
+      "grad_norm": 3.4996840953826904,
+      "learning_rate": 1.367619926199262e-06,
+      "loss": 1.4069,
+      "step": 1832
+    },
+    {
+      "epoch": 2.281269446172993,
+      "grad_norm": 3.4036638736724854,
+      "learning_rate": 1.3653136531365315e-06,
+      "loss": 1.3499,
+      "step": 1833
+    },
+    {
+      "epoch": 2.282514001244555,
+      "grad_norm": 3.328805685043335,
+      "learning_rate": 1.363007380073801e-06,
+      "loss": 1.3266,
+      "step": 1834
+    },
+    {
+      "epoch": 2.283758556316117,
+      "grad_norm": 3.410924196243286,
+      "learning_rate": 1.3607011070110701e-06,
+      "loss": 1.3303,
+      "step": 1835
+    },
+    {
+      "epoch": 2.2850031113876788,
+      "grad_norm": 3.323162794113159,
+      "learning_rate": 1.3583948339483396e-06,
+      "loss": 1.3284,
+      "step": 1836
+    },
+    {
+      "epoch": 2.286247666459241,
+      "grad_norm": 3.3356032371520996,
+      "learning_rate": 1.356088560885609e-06,
+      "loss": 1.2842,
+      "step": 1837
+    },
+    {
+      "epoch": 2.287492221530803,
+      "grad_norm": 3.3366599082946777,
+      "learning_rate": 1.3537822878228784e-06,
+      "loss": 1.321,
+      "step": 1838
+    },
+    {
+      "epoch": 2.2887367766023647,
+      "grad_norm": 3.3824799060821533,
+      "learning_rate": 1.3514760147601478e-06,
+      "loss": 1.3744,
+      "step": 1839
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "grad_norm": 3.6268486976623535,
+      "learning_rate": 1.349169741697417e-06,
+      "loss": 1.4097,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "eval_loss": 1.6148018836975098,
+      "eval_runtime": 48.3459,
+      "eval_samples_per_second": 20.684,
+      "eval_steps_per_second": 0.869,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2912258867454884,
+      "grad_norm": 3.3530380725860596,
+      "learning_rate": 1.3468634686346865e-06,
+      "loss": 1.3924,
+      "step": 1841
+    },
+    {
+      "epoch": 2.2924704418170503,
+      "grad_norm": 3.456149101257324,
+      "learning_rate": 1.3445571955719559e-06,
+      "loss": 1.3837,
+      "step": 1842
+    },
+    {
+      "epoch": 2.293714996888612,
+      "grad_norm": 3.37180495262146,
+      "learning_rate": 1.3422509225092253e-06,
+      "loss": 1.3411,
+      "step": 1843
+    },
+    {
+      "epoch": 2.2949595519601744,
+      "grad_norm": 3.286172389984131,
+      "learning_rate": 1.3399446494464945e-06,
+      "loss": 1.2888,
+      "step": 1844
+    },
+    {
+      "epoch": 2.2962041070317363,
+      "grad_norm": 3.3966121673583984,
+      "learning_rate": 1.337638376383764e-06,
+      "loss": 1.3644,
+      "step": 1845
+    },
+    {
+      "epoch": 2.297448662103298,
+      "grad_norm": 3.4066531658172607,
+      "learning_rate": 1.3353321033210334e-06,
+      "loss": 1.3543,
+      "step": 1846
+    },
+    {
+      "epoch": 2.29869321717486,
+      "grad_norm": 3.542680501937866,
+      "learning_rate": 1.3330258302583028e-06,
+      "loss": 1.3823,
+      "step": 1847
+    },
+    {
+      "epoch": 2.299937772246422,
+      "grad_norm": 3.323636770248413,
+      "learning_rate": 1.330719557195572e-06,
+      "loss": 1.3031,
+      "step": 1848
+    },
+    {
+      "epoch": 2.3011823273179837,
+      "grad_norm": 3.4802908897399902,
+      "learning_rate": 1.3284132841328414e-06,
+      "loss": 1.4001,
+      "step": 1849
+    },
+    {
+      "epoch": 2.302426882389546,
+      "grad_norm": 3.310987710952759,
+      "learning_rate": 1.3261070110701108e-06,
+      "loss": 1.3385,
+      "step": 1850
+    },
+    {
+      "epoch": 2.302426882389546,
+      "eval_loss": 1.615319848060608,
+      "eval_runtime": 44.8343,
+      "eval_samples_per_second": 22.304,
+      "eval_steps_per_second": 0.937,
+      "step": 1850
+    },
+    {
+      "epoch": 2.303671437461108,
+      "grad_norm": 3.3461661338806152,
+      "learning_rate": 1.3238007380073803e-06,
+      "loss": 1.3461,
+      "step": 1851
+    },
+    {
+      "epoch": 2.3049159925326697,
+      "grad_norm": 3.317770481109619,
+      "learning_rate": 1.3214944649446495e-06,
+      "loss": 1.338,
+      "step": 1852
+    },
+    {
+      "epoch": 2.3061605476042315,
+      "grad_norm": 3.3806679248809814,
+      "learning_rate": 1.319188191881919e-06,
+      "loss": 1.3367,
+      "step": 1853
+    },
+    {
+      "epoch": 2.3074051026757934,
+      "grad_norm": 3.366933822631836,
+      "learning_rate": 1.3168819188191883e-06,
+      "loss": 1.3717,
+      "step": 1854
+    },
+    {
+      "epoch": 2.308649657747355,
+      "grad_norm": 3.3250250816345215,
+      "learning_rate": 1.3145756457564575e-06,
+      "loss": 1.3585,
+      "step": 1855
+    },
+    {
+      "epoch": 2.309894212818917,
+      "grad_norm": 3.320171356201172,
+      "learning_rate": 1.3122693726937272e-06,
+      "loss": 1.3491,
+      "step": 1856
+    },
+    {
+      "epoch": 2.3111387678904793,
+      "grad_norm": 3.418029546737671,
+      "learning_rate": 1.3099630996309964e-06,
+      "loss": 1.3627,
+      "step": 1857
+    },
+    {
+      "epoch": 2.312383322962041,
+      "grad_norm": 3.3762004375457764,
+      "learning_rate": 1.3076568265682658e-06,
+      "loss": 1.3409,
+      "step": 1858
+    },
+    {
+      "epoch": 2.313627878033603,
+      "grad_norm": 3.3099405765533447,
+      "learning_rate": 1.3053505535055352e-06,
+      "loss": 1.3593,
+      "step": 1859
+    },
+    {
+      "epoch": 2.314872433105165,
+      "grad_norm": 3.4060535430908203,
+      "learning_rate": 1.3030442804428044e-06,
+      "loss": 1.3363,
+      "step": 1860
+    },
+    {
+      "epoch": 2.314872433105165,
+      "eval_loss": 1.6176154613494873,
+      "eval_runtime": 46.8022,
+      "eval_samples_per_second": 21.367,
+      "eval_steps_per_second": 0.897,
+      "step": 1860
+    },
+    {
+      "epoch": 2.3161169881767267,
+      "grad_norm": 3.2827699184417725,
+      "learning_rate": 1.3007380073800739e-06,
+      "loss": 1.2718,
+      "step": 1861
+    },
+    {
+      "epoch": 2.3173615432482886,
+      "grad_norm": 3.5225489139556885,
+      "learning_rate": 1.2984317343173433e-06,
+      "loss": 1.3728,
+      "step": 1862
+    },
+    {
+      "epoch": 2.318606098319851,
+      "grad_norm": 3.475665807723999,
+      "learning_rate": 1.2961254612546125e-06,
+      "loss": 1.3772,
+      "step": 1863
+    },
+    {
+      "epoch": 2.3198506533914127,
+      "grad_norm": 3.385239839553833,
+      "learning_rate": 1.2938191881918821e-06,
+      "loss": 1.3471,
+      "step": 1864
+    },
+    {
+      "epoch": 2.3210952084629746,
+      "grad_norm": 3.312469005584717,
+      "learning_rate": 1.2915129151291513e-06,
+      "loss": 1.2961,
+      "step": 1865
+    },
+    {
+      "epoch": 2.3223397635345364,
+      "grad_norm": 3.3883018493652344,
+      "learning_rate": 1.2892066420664208e-06,
+      "loss": 1.3778,
+      "step": 1866
+    },
+    {
+      "epoch": 2.3235843186060983,
+      "grad_norm": 3.368964195251465,
+      "learning_rate": 1.2869003690036902e-06,
+      "loss": 1.3153,
+      "step": 1867
+    },
+    {
+      "epoch": 2.32482887367766,
+      "grad_norm": 3.4132297039031982,
+      "learning_rate": 1.2845940959409594e-06,
+      "loss": 1.3718,
+      "step": 1868
+    },
+    {
+      "epoch": 2.326073428749222,
+      "grad_norm": 3.3750340938568115,
+      "learning_rate": 1.282287822878229e-06,
+      "loss": 1.4035,
+      "step": 1869
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "grad_norm": 3.372588872909546,
+      "learning_rate": 1.2799815498154982e-06,
+      "loss": 1.3591,
+      "step": 1870
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "eval_loss": 1.6196452379226685,
+      "eval_runtime": 43.8125,
+      "eval_samples_per_second": 22.825,
+      "eval_steps_per_second": 0.959,
+      "step": 1870
+    },
+    {
+      "epoch": 2.328562538892346,
+      "grad_norm": 3.341944694519043,
+      "learning_rate": 1.2776752767527675e-06,
+      "loss": 1.3239,
+      "step": 1871
+    },
+    {
+      "epoch": 2.329807093963908,
+      "grad_norm": 3.4855759143829346,
+      "learning_rate": 1.275369003690037e-06,
+      "loss": 1.3712,
+      "step": 1872
+    },
+    {
+      "epoch": 2.33105164903547,
+      "grad_norm": 3.577047348022461,
+      "learning_rate": 1.2730627306273063e-06,
+      "loss": 1.3502,
+      "step": 1873
+    },
+    {
+      "epoch": 2.3322962041070316,
+      "grad_norm": 3.479374885559082,
+      "learning_rate": 1.270756457564576e-06,
+      "loss": 1.3296,
+      "step": 1874
+    },
+    {
+      "epoch": 2.3335407591785935,
+      "grad_norm": 3.2203609943389893,
+      "learning_rate": 1.2684501845018451e-06,
+      "loss": 1.3105,
+      "step": 1875
+    },
+    {
+      "epoch": 2.334785314250156,
+      "grad_norm": 3.3177664279937744,
+      "learning_rate": 1.2661439114391144e-06,
+      "loss": 1.3365,
+      "step": 1876
+    },
+    {
+      "epoch": 2.3360298693217176,
+      "grad_norm": 3.224146842956543,
+      "learning_rate": 1.263837638376384e-06,
+      "loss": 1.3172,
+      "step": 1877
+    },
+    {
+      "epoch": 2.3372744243932795,
+      "grad_norm": 3.336106538772583,
+      "learning_rate": 1.2615313653136532e-06,
+      "loss": 1.3248,
+      "step": 1878
+    },
+    {
+      "epoch": 2.3385189794648413,
+      "grad_norm": 3.4414947032928467,
+      "learning_rate": 1.2592250922509224e-06,
+      "loss": 1.3637,
+      "step": 1879
+    },
+    {
+      "epoch": 2.339763534536403,
+      "grad_norm": 3.5528085231781006,
+      "learning_rate": 1.256918819188192e-06,
+      "loss": 1.3986,
+      "step": 1880
+    },
+    {
+      "epoch": 2.339763534536403,
+      "eval_loss": 1.6137934923171997,
+      "eval_runtime": 43.8859,
+      "eval_samples_per_second": 22.786,
+      "eval_steps_per_second": 0.957,
+      "step": 1880
+    },
+    {
+      "epoch": 2.341008089607965,
+      "grad_norm": 3.406446695327759,
+      "learning_rate": 1.2546125461254613e-06,
+      "loss": 1.3479,
+      "step": 1881
+    },
+    {
+      "epoch": 2.342252644679527,
+      "grad_norm": 3.4129297733306885,
+      "learning_rate": 1.2523062730627309e-06,
+      "loss": 1.3227,
+      "step": 1882
+    },
+    {
+      "epoch": 2.343497199751089,
+      "grad_norm": 3.3013577461242676,
+      "learning_rate": 1.25e-06,
+      "loss": 1.3277,
+      "step": 1883
+    },
+    {
+      "epoch": 2.344741754822651,
+      "grad_norm": 3.3039028644561768,
+      "learning_rate": 1.2476937269372695e-06,
+      "loss": 1.3208,
+      "step": 1884
+    },
+    {
+      "epoch": 2.345986309894213,
+      "grad_norm": 3.3239917755126953,
+      "learning_rate": 1.245387453874539e-06,
+      "loss": 1.3293,
+      "step": 1885
+    },
+    {
+      "epoch": 2.3472308649657747,
+      "grad_norm": 3.239821195602417,
+      "learning_rate": 1.2430811808118082e-06,
+      "loss": 1.3191,
+      "step": 1886
+    },
+    {
+      "epoch": 2.3484754200373366,
+      "grad_norm": 3.6869564056396484,
+      "learning_rate": 1.2407749077490776e-06,
+      "loss": 1.363,
+      "step": 1887
+    },
+    {
+      "epoch": 2.3497199751088984,
+      "grad_norm": 3.5775156021118164,
+      "learning_rate": 1.238468634686347e-06,
+      "loss": 1.3564,
+      "step": 1888
+    },
+    {
+      "epoch": 2.3509645301804607,
+      "grad_norm": 3.3177807331085205,
+      "learning_rate": 1.2361623616236164e-06,
+      "loss": 1.3044,
+      "step": 1889
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "grad_norm": 3.622527837753296,
+      "learning_rate": 1.2338560885608856e-06,
+      "loss": 1.381,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "eval_loss": 1.6165976524353027,
+      "eval_runtime": 44.4026,
+      "eval_samples_per_second": 22.521,
+      "eval_steps_per_second": 0.946,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3534536403235844,
+      "grad_norm": 3.3163886070251465,
+      "learning_rate": 1.231549815498155e-06,
+      "loss": 1.3598,
+      "step": 1891
+    },
+    {
+      "epoch": 2.3546981953951462,
+      "grad_norm": 3.3344099521636963,
+      "learning_rate": 1.2292435424354245e-06,
+      "loss": 1.3733,
+      "step": 1892
+    },
+    {
+      "epoch": 2.355942750466708,
+      "grad_norm": 3.259413480758667,
+      "learning_rate": 1.226937269372694e-06,
+      "loss": 1.3281,
+      "step": 1893
+    },
+    {
+      "epoch": 2.35718730553827,
+      "grad_norm": 3.427147150039673,
+      "learning_rate": 1.2246309963099633e-06,
+      "loss": 1.3484,
+      "step": 1894
+    },
+    {
+      "epoch": 2.358431860609832,
+      "grad_norm": 3.3890154361724854,
+      "learning_rate": 1.2223247232472325e-06,
+      "loss": 1.3768,
+      "step": 1895
+    },
+    {
+      "epoch": 2.359676415681394,
+      "grad_norm": 3.2845499515533447,
+      "learning_rate": 1.220018450184502e-06,
+      "loss": 1.3579,
+      "step": 1896
+    },
+    {
+      "epoch": 2.360920970752956,
+      "grad_norm": 3.3046178817749023,
+      "learning_rate": 1.2177121771217714e-06,
+      "loss": 1.3322,
+      "step": 1897
+    },
+    {
+      "epoch": 2.3621655258245178,
+      "grad_norm": 3.4520273208618164,
+      "learning_rate": 1.2154059040590408e-06,
+      "loss": 1.3515,
+      "step": 1898
+    },
+    {
+      "epoch": 2.3634100808960796,
+      "grad_norm": 3.3665220737457275,
+      "learning_rate": 1.21309963099631e-06,
+      "loss": 1.3236,
+      "step": 1899
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "grad_norm": 3.3551578521728516,
+      "learning_rate": 1.2107933579335794e-06,
+      "loss": 1.3568,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "eval_loss": 1.6159367561340332,
+      "eval_runtime": 44.1216,
+      "eval_samples_per_second": 22.665,
+      "eval_steps_per_second": 0.952,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3658991910392033,
+      "grad_norm": 3.6204397678375244,
+      "learning_rate": 1.2084870848708489e-06,
+      "loss": 1.4072,
+      "step": 1901
+    },
+    {
+      "epoch": 2.3671437461107656,
+      "grad_norm": 3.3525922298431396,
+      "learning_rate": 1.2061808118081183e-06,
+      "loss": 1.328,
+      "step": 1902
+    },
+    {
+      "epoch": 2.3683883011823275,
+      "grad_norm": 3.3686022758483887,
+      "learning_rate": 1.2038745387453875e-06,
+      "loss": 1.3086,
+      "step": 1903
+    },
+    {
+      "epoch": 2.3696328562538893,
+      "grad_norm": 3.5756561756134033,
+      "learning_rate": 1.201568265682657e-06,
+      "loss": 1.3784,
+      "step": 1904
+    },
+    {
+      "epoch": 2.370877411325451,
+      "grad_norm": 3.4186534881591797,
+      "learning_rate": 1.1992619926199263e-06,
+      "loss": 1.3546,
+      "step": 1905
+    },
+    {
+      "epoch": 2.372121966397013,
+      "grad_norm": 3.367868661880493,
+      "learning_rate": 1.1969557195571958e-06,
+      "loss": 1.3527,
+      "step": 1906
+    },
+    {
+      "epoch": 2.373366521468575,
+      "grad_norm": 3.3395912647247314,
+      "learning_rate": 1.1946494464944652e-06,
+      "loss": 1.3366,
+      "step": 1907
+    },
+    {
+      "epoch": 2.3746110765401367,
+      "grad_norm": 3.2967982292175293,
+      "learning_rate": 1.1923431734317344e-06,
+      "loss": 1.3229,
+      "step": 1908
+    },
+    {
+      "epoch": 2.375855631611699,
+      "grad_norm": 3.2422475814819336,
+      "learning_rate": 1.1900369003690038e-06,
+      "loss": 1.3455,
+      "step": 1909
+    },
+    {
+      "epoch": 2.377100186683261,
+      "grad_norm": 3.3781206607818604,
+      "learning_rate": 1.1877306273062733e-06,
+      "loss": 1.3314,
+      "step": 1910
+    },
+    {
+      "epoch": 2.377100186683261,
+      "eval_loss": 1.6179407835006714,
+      "eval_runtime": 42.9089,
+      "eval_samples_per_second": 23.305,
+      "eval_steps_per_second": 0.979,
+      "step": 1910
+    },
+    {
+      "epoch": 2.3783447417548227,
+      "grad_norm": 3.438586711883545,
+      "learning_rate": 1.1854243542435427e-06,
+      "loss": 1.3654,
+      "step": 1911
+    },
+    {
+      "epoch": 2.3795892968263845,
+      "grad_norm": 3.503896474838257,
+      "learning_rate": 1.1831180811808119e-06,
+      "loss": 1.3716,
+      "step": 1912
+    },
+    {
+      "epoch": 2.3808338518979464,
+      "grad_norm": 3.411320686340332,
+      "learning_rate": 1.1808118081180813e-06,
+      "loss": 1.3315,
+      "step": 1913
+    },
+    {
+      "epoch": 2.3820784069695082,
+      "grad_norm": 3.1860697269439697,
+      "learning_rate": 1.1785055350553507e-06,
+      "loss": 1.3016,
+      "step": 1914
+    },
+    {
+      "epoch": 2.3833229620410705,
+      "grad_norm": 3.4494664669036865,
+      "learning_rate": 1.1761992619926202e-06,
+      "loss": 1.3629,
+      "step": 1915
+    },
+    {
+      "epoch": 2.3845675171126324,
+      "grad_norm": 3.303277015686035,
+      "learning_rate": 1.1738929889298894e-06,
+      "loss": 1.3207,
+      "step": 1916
+    },
+    {
+      "epoch": 2.385812072184194,
+      "grad_norm": 3.254411458969116,
+      "learning_rate": 1.1715867158671588e-06,
+      "loss": 1.3477,
+      "step": 1917
+    },
+    {
+      "epoch": 2.387056627255756,
+      "grad_norm": 3.258666753768921,
+      "learning_rate": 1.1692804428044282e-06,
+      "loss": 1.3452,
+      "step": 1918
+    },
+    {
+      "epoch": 2.388301182327318,
+      "grad_norm": 3.3814001083374023,
+      "learning_rate": 1.1669741697416974e-06,
+      "loss": 1.329,
+      "step": 1919
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "grad_norm": 3.367011547088623,
+      "learning_rate": 1.1646678966789668e-06,
+      "loss": 1.3326,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "eval_loss": 1.6158324480056763,
+      "eval_runtime": 45.8088,
+      "eval_samples_per_second": 21.83,
+      "eval_steps_per_second": 0.917,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3907902924704416,
+      "grad_norm": 3.313123941421509,
+      "learning_rate": 1.1623616236162363e-06,
+      "loss": 1.307,
+      "step": 1921
+    },
+    {
+      "epoch": 2.392034847542004,
+      "grad_norm": 3.27423357963562,
+      "learning_rate": 1.1600553505535057e-06,
+      "loss": 1.3249,
+      "step": 1922
+    },
+    {
+      "epoch": 2.3932794026135658,
+      "grad_norm": 3.2963404655456543,
+      "learning_rate": 1.157749077490775e-06,
+      "loss": 1.3394,
+      "step": 1923
+    },
+    {
+      "epoch": 2.3945239576851276,
+      "grad_norm": 3.387673854827881,
+      "learning_rate": 1.1554428044280443e-06,
+      "loss": 1.3586,
+      "step": 1924
+    },
+    {
+      "epoch": 2.3957685127566895,
+      "grad_norm": 3.2588143348693848,
+      "learning_rate": 1.1531365313653137e-06,
+      "loss": 1.317,
+      "step": 1925
+    },
+    {
+      "epoch": 2.3970130678282513,
+      "grad_norm": 3.285653829574585,
+      "learning_rate": 1.1508302583025832e-06,
+      "loss": 1.2847,
+      "step": 1926
+    },
+    {
+      "epoch": 2.398257622899813,
+      "grad_norm": 3.4495935440063477,
+      "learning_rate": 1.1485239852398524e-06,
+      "loss": 1.3815,
+      "step": 1927
+    },
+    {
+      "epoch": 2.3995021779713754,
+      "grad_norm": 3.287186622619629,
+      "learning_rate": 1.1462177121771218e-06,
+      "loss": 1.3048,
+      "step": 1928
+    },
+    {
+      "epoch": 2.4007467330429373,
+      "grad_norm": 3.318042278289795,
+      "learning_rate": 1.1439114391143912e-06,
+      "loss": 1.313,
+      "step": 1929
+    },
+    {
+      "epoch": 2.401991288114499,
+      "grad_norm": 3.2705087661743164,
+      "learning_rate": 1.1416051660516606e-06,
+      "loss": 1.3207,
+      "step": 1930
+    },
+    {
+      "epoch": 2.401991288114499,
+      "eval_loss": 1.613088846206665,
+      "eval_runtime": 46.0901,
+      "eval_samples_per_second": 21.697,
+      "eval_steps_per_second": 0.911,
+      "step": 1930
+    },
+    {
+      "epoch": 2.403235843186061,
+      "grad_norm": 3.307523250579834,
+      "learning_rate": 1.1392988929889299e-06,
+      "loss": 1.3312,
+      "step": 1931
+    },
+    {
+      "epoch": 2.404480398257623,
+      "grad_norm": 3.3040900230407715,
+      "learning_rate": 1.1369926199261993e-06,
+      "loss": 1.3279,
+      "step": 1932
+    },
+    {
+      "epoch": 2.4057249533291847,
+      "grad_norm": 3.232649087905884,
+      "learning_rate": 1.1346863468634687e-06,
+      "loss": 1.314,
+      "step": 1933
+    },
+    {
+      "epoch": 2.4069695084007465,
+      "grad_norm": 3.3581860065460205,
+      "learning_rate": 1.1323800738007381e-06,
+      "loss": 1.3158,
+      "step": 1934
+    },
+    {
+      "epoch": 2.408214063472309,
+      "grad_norm": 3.378774404525757,
+      "learning_rate": 1.1300738007380073e-06,
+      "loss": 1.3252,
+      "step": 1935
+    },
+    {
+      "epoch": 2.4094586185438707,
+      "grad_norm": 3.4180240631103516,
+      "learning_rate": 1.1277675276752768e-06,
+      "loss": 1.3606,
+      "step": 1936
+    },
+    {
+      "epoch": 2.4107031736154325,
+      "grad_norm": 3.410921335220337,
+      "learning_rate": 1.1254612546125462e-06,
+      "loss": 1.3484,
+      "step": 1937
+    },
+    {
+      "epoch": 2.4119477286869944,
+      "grad_norm": 3.4201443195343018,
+      "learning_rate": 1.1231549815498156e-06,
+      "loss": 1.3659,
+      "step": 1938
+    },
+    {
+      "epoch": 2.413192283758556,
+      "grad_norm": 3.381019115447998,
+      "learning_rate": 1.1208487084870848e-06,
+      "loss": 1.3261,
+      "step": 1939
+    },
+    {
+      "epoch": 2.414436838830118,
+      "grad_norm": 3.516425371170044,
+      "learning_rate": 1.1185424354243542e-06,
+      "loss": 1.3618,
+      "step": 1940
+    },
+    {
+      "epoch": 2.414436838830118,
+      "eval_loss": 1.6142985820770264,
+      "eval_runtime": 45.8868,
+      "eval_samples_per_second": 21.793,
+      "eval_steps_per_second": 0.915,
+      "step": 1940
+    },
+    {
+      "epoch": 2.4156813939016804,
+      "grad_norm": 3.5319905281066895,
+      "learning_rate": 1.1162361623616237e-06,
+      "loss": 1.338,
+      "step": 1941
+    },
+    {
+      "epoch": 2.416925948973242,
+      "grad_norm": 3.4355592727661133,
+      "learning_rate": 1.113929889298893e-06,
+      "loss": 1.3389,
+      "step": 1942
+    },
+    {
+      "epoch": 2.418170504044804,
+      "grad_norm": 3.362187623977661,
+      "learning_rate": 1.1116236162361623e-06,
+      "loss": 1.3194,
+      "step": 1943
+    },
+    {
+      "epoch": 2.419415059116366,
+      "grad_norm": 3.4991087913513184,
+      "learning_rate": 1.1093173431734317e-06,
+      "loss": 1.3758,
+      "step": 1944
+    },
+    {
+      "epoch": 2.4206596141879277,
+      "grad_norm": 3.5951290130615234,
+      "learning_rate": 1.1070110701107011e-06,
+      "loss": 1.3548,
+      "step": 1945
+    },
+    {
+      "epoch": 2.4219041692594896,
+      "grad_norm": 3.3458032608032227,
+      "learning_rate": 1.1047047970479706e-06,
+      "loss": 1.3483,
+      "step": 1946
+    },
+    {
+      "epoch": 2.4231487243310514,
+      "grad_norm": 3.426600933074951,
+      "learning_rate": 1.10239852398524e-06,
+      "loss": 1.3548,
+      "step": 1947
+    },
+    {
+      "epoch": 2.4243932794026137,
+      "grad_norm": 3.5074410438537598,
+      "learning_rate": 1.1000922509225092e-06,
+      "loss": 1.3608,
+      "step": 1948
+    },
+    {
+      "epoch": 2.4256378344741756,
+      "grad_norm": 3.5622129440307617,
+      "learning_rate": 1.0977859778597786e-06,
+      "loss": 1.3861,
+      "step": 1949
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "grad_norm": 3.537771463394165,
+      "learning_rate": 1.095479704797048e-06,
+      "loss": 1.3408,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "eval_loss": 1.612324833869934,
+      "eval_runtime": 46.7327,
+      "eval_samples_per_second": 21.398,
+      "eval_steps_per_second": 0.899,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4281269446172993,
+      "grad_norm": 3.430544137954712,
+      "learning_rate": 1.0931734317343175e-06,
+      "loss": 1.3678,
+      "step": 1951
+    },
+    {
+      "epoch": 2.429371499688861,
+      "grad_norm": 3.2243361473083496,
+      "learning_rate": 1.0908671586715867e-06,
+      "loss": 1.2982,
+      "step": 1952
+    },
+    {
+      "epoch": 2.430616054760423,
+      "grad_norm": 3.426236867904663,
+      "learning_rate": 1.088560885608856e-06,
+      "loss": 1.3827,
+      "step": 1953
+    },
+    {
+      "epoch": 2.4318606098319853,
+      "grad_norm": 3.3180603981018066,
+      "learning_rate": 1.0862546125461255e-06,
+      "loss": 1.3065,
+      "step": 1954
+    },
+    {
+      "epoch": 2.433105164903547,
+      "grad_norm": 3.3775084018707275,
+      "learning_rate": 1.083948339483395e-06,
+      "loss": 1.2759,
+      "step": 1955
+    },
+    {
+      "epoch": 2.434349719975109,
+      "grad_norm": 3.4124209880828857,
+      "learning_rate": 1.0816420664206644e-06,
+      "loss": 1.3571,
+      "step": 1956
+    },
+    {
+      "epoch": 2.435594275046671,
+      "grad_norm": 3.494408369064331,
+      "learning_rate": 1.0793357933579336e-06,
+      "loss": 1.3931,
+      "step": 1957
+    },
+    {
+      "epoch": 2.4368388301182327,
+      "grad_norm": 3.2699358463287354,
+      "learning_rate": 1.077029520295203e-06,
+      "loss": 1.3182,
+      "step": 1958
+    },
+    {
+      "epoch": 2.4380833851897945,
+      "grad_norm": 3.281592607498169,
+      "learning_rate": 1.0747232472324724e-06,
+      "loss": 1.2629,
+      "step": 1959
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "grad_norm": 3.3045406341552734,
+      "learning_rate": 1.0724169741697419e-06,
+      "loss": 1.3093,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "eval_loss": 1.6135376691818237,
+      "eval_runtime": 51.0587,
+      "eval_samples_per_second": 19.585,
+      "eval_steps_per_second": 0.823,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4405724953329186,
+      "grad_norm": 3.3883392810821533,
+      "learning_rate": 1.070110701107011e-06,
+      "loss": 1.4005,
+      "step": 1961
+    },
+    {
+      "epoch": 2.4418170504044805,
+      "grad_norm": 3.3957419395446777,
+      "learning_rate": 1.0678044280442805e-06,
+      "loss": 1.3342,
+      "step": 1962
+    },
+    {
+      "epoch": 2.4430616054760423,
+      "grad_norm": 3.1982152462005615,
+      "learning_rate": 1.06549815498155e-06,
+      "loss": 1.3273,
+      "step": 1963
+    },
+    {
+      "epoch": 2.444306160547604,
+      "grad_norm": 3.3369827270507812,
+      "learning_rate": 1.0631918819188193e-06,
+      "loss": 1.312,
+      "step": 1964
+    },
+    {
+      "epoch": 2.445550715619166,
+      "grad_norm": 3.3651299476623535,
+      "learning_rate": 1.0608856088560888e-06,
+      "loss": 1.3728,
+      "step": 1965
+    },
+    {
+      "epoch": 2.446795270690728,
+      "grad_norm": 3.3393259048461914,
+      "learning_rate": 1.058579335793358e-06,
+      "loss": 1.3454,
+      "step": 1966
+    },
+    {
+      "epoch": 2.44803982576229,
+      "grad_norm": 3.4974067211151123,
+      "learning_rate": 1.0562730627306274e-06,
+      "loss": 1.3747,
+      "step": 1967
+    },
+    {
+      "epoch": 2.449284380833852,
+      "grad_norm": 3.4064528942108154,
+      "learning_rate": 1.0539667896678968e-06,
+      "loss": 1.3695,
+      "step": 1968
+    },
+    {
+      "epoch": 2.450528935905414,
+      "grad_norm": 3.347919225692749,
+      "learning_rate": 1.0516605166051662e-06,
+      "loss": 1.3439,
+      "step": 1969
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "grad_norm": 3.4505932331085205,
+      "learning_rate": 1.0493542435424354e-06,
+      "loss": 1.3723,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "eval_loss": 1.611203908920288,
+      "eval_runtime": 52.5131,
+      "eval_samples_per_second": 19.043,
+      "eval_steps_per_second": 0.8,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4530180460485376,
+      "grad_norm": 3.4287805557250977,
+      "learning_rate": 1.0470479704797049e-06,
+      "loss": 1.3426,
+      "step": 1971
+    },
+    {
+      "epoch": 2.4542626011200994,
+      "grad_norm": 3.4337522983551025,
+      "learning_rate": 1.0447416974169743e-06,
+      "loss": 1.3451,
+      "step": 1972
+    },
+    {
+      "epoch": 2.4555071561916613,
+      "grad_norm": 3.4307212829589844,
+      "learning_rate": 1.0424354243542437e-06,
+      "loss": 1.3097,
+      "step": 1973
+    },
+    {
+      "epoch": 2.4567517112632236,
+      "grad_norm": 3.307767868041992,
+      "learning_rate": 1.0401291512915131e-06,
+      "loss": 1.2751,
+      "step": 1974
+    },
+    {
+      "epoch": 2.4579962663347854,
+      "grad_norm": 3.4809703826904297,
+      "learning_rate": 1.0378228782287824e-06,
+      "loss": 1.3522,
+      "step": 1975
+    },
+    {
+      "epoch": 2.4592408214063473,
+      "grad_norm": 3.235170602798462,
+      "learning_rate": 1.0355166051660518e-06,
+      "loss": 1.3003,
+      "step": 1976
+    },
+    {
+      "epoch": 2.460485376477909,
+      "grad_norm": 3.2969000339508057,
+      "learning_rate": 1.0332103321033212e-06,
+      "loss": 1.2993,
+      "step": 1977
+    },
+    {
+      "epoch": 2.461729931549471,
+      "grad_norm": 3.311535358428955,
+      "learning_rate": 1.0309040590405906e-06,
+      "loss": 1.3262,
+      "step": 1978
+    },
+    {
+      "epoch": 2.462974486621033,
+      "grad_norm": 3.2320244312286377,
+      "learning_rate": 1.0285977859778598e-06,
+      "loss": 1.3111,
+      "step": 1979
+    },
+    {
+      "epoch": 2.464219041692595,
+      "grad_norm": 3.2673771381378174,
+      "learning_rate": 1.0262915129151293e-06,
+      "loss": 1.3439,
+      "step": 1980
+    },
+    {
+      "epoch": 2.464219041692595,
+      "eval_loss": 1.6103569269180298,
+      "eval_runtime": 47.2563,
+      "eval_samples_per_second": 21.161,
+      "eval_steps_per_second": 0.889,
+      "step": 1980
+    },
+    {
+      "epoch": 2.465463596764157,
+      "grad_norm": 3.324113130569458,
+      "learning_rate": 1.0239852398523987e-06,
+      "loss": 1.3278,
+      "step": 1981
+    },
+    {
+      "epoch": 2.466708151835719,
+      "grad_norm": 3.3066444396972656,
+      "learning_rate": 1.021678966789668e-06,
+      "loss": 1.3438,
+      "step": 1982
+    },
+    {
+      "epoch": 2.4679527069072806,
+      "grad_norm": 3.463088035583496,
+      "learning_rate": 1.0193726937269373e-06,
+      "loss": 1.3293,
+      "step": 1983
+    },
+    {
+      "epoch": 2.4691972619788425,
+      "grad_norm": 3.406008005142212,
+      "learning_rate": 1.0170664206642067e-06,
+      "loss": 1.4026,
+      "step": 1984
+    },
+    {
+      "epoch": 2.4704418170504043,
+      "grad_norm": 3.408823251724243,
+      "learning_rate": 1.0147601476014762e-06,
+      "loss": 1.3226,
+      "step": 1985
+    },
+    {
+      "epoch": 2.471686372121966,
+      "grad_norm": 3.2991373538970947,
+      "learning_rate": 1.0124538745387456e-06,
+      "loss": 1.2682,
+      "step": 1986
+    },
+    {
+      "epoch": 2.4729309271935285,
+      "grad_norm": 3.3868234157562256,
+      "learning_rate": 1.010147601476015e-06,
+      "loss": 1.3286,
+      "step": 1987
+    },
+    {
+      "epoch": 2.4741754822650903,
+      "grad_norm": 3.3641555309295654,
+      "learning_rate": 1.0078413284132842e-06,
+      "loss": 1.3423,
+      "step": 1988
+    },
+    {
+      "epoch": 2.475420037336652,
+      "grad_norm": 3.3237619400024414,
+      "learning_rate": 1.0055350553505536e-06,
+      "loss": 1.2979,
+      "step": 1989
+    },
+    {
+      "epoch": 2.476664592408214,
+      "grad_norm": 3.6278553009033203,
+      "learning_rate": 1.003228782287823e-06,
+      "loss": 1.3543,
+      "step": 1990
+    },
+    {
+      "epoch": 2.476664592408214,
+      "eval_loss": 1.6103031635284424,
+      "eval_runtime": 46.0573,
+      "eval_samples_per_second": 21.712,
+      "eval_steps_per_second": 0.912,
+      "step": 1990
+    },
+    {
+      "epoch": 2.477909147479776,
+      "grad_norm": 3.2861995697021484,
+      "learning_rate": 1.0009225092250923e-06,
+      "loss": 1.3303,
+      "step": 1991
+    },
+    {
+      "epoch": 2.4791537025513377,
+      "grad_norm": 3.352261781692505,
+      "learning_rate": 9.986162361623617e-07,
+      "loss": 1.3527,
+      "step": 1992
+    },
+    {
+      "epoch": 2.4803982576229,
+      "grad_norm": 3.483726978302002,
+      "learning_rate": 9.963099630996311e-07,
+      "loss": 1.3762,
+      "step": 1993
+    },
+    {
+      "epoch": 2.481642812694462,
+      "grad_norm": 3.357154607772827,
+      "learning_rate": 9.940036900369005e-07,
+      "loss": 1.3601,
+      "step": 1994
+    },
+    {
+      "epoch": 2.4828873677660237,
+      "grad_norm": 3.4477577209472656,
+      "learning_rate": 9.916974169741698e-07,
+      "loss": 1.3636,
+      "step": 1995
+    },
+    {
+      "epoch": 2.4841319228375855,
+      "grad_norm": 3.377279043197632,
+      "learning_rate": 9.893911439114392e-07,
+      "loss": 1.3349,
+      "step": 1996
+    },
+    {
+      "epoch": 2.4853764779091474,
+      "grad_norm": 3.2705130577087402,
+      "learning_rate": 9.870848708487086e-07,
+      "loss": 1.3659,
+      "step": 1997
+    },
+    {
+      "epoch": 2.4866210329807092,
+      "grad_norm": 3.418225049972534,
+      "learning_rate": 9.84778597785978e-07,
+      "loss": 1.3534,
+      "step": 1998
+    },
+    {
+      "epoch": 2.487865588052271,
+      "grad_norm": 3.2288014888763428,
+      "learning_rate": 9.824723247232472e-07,
+      "loss": 1.3392,
+      "step": 1999
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "grad_norm": 3.3513882160186768,
+      "learning_rate": 9.801660516605167e-07,
+      "loss": 1.3483,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "eval_loss": 1.6108943223953247,
+      "eval_runtime": 48.275,
+      "eval_samples_per_second": 20.715,
+      "eval_steps_per_second": 0.87,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4903546981953952,
+      "grad_norm": 3.35583758354187,
+      "learning_rate": 9.77859778597786e-07,
+      "loss": 1.3808,
+      "step": 2001
+    },
+    {
+      "epoch": 2.491599253266957,
+      "grad_norm": 3.4611411094665527,
+      "learning_rate": 9.755535055350555e-07,
+      "loss": 1.3512,
+      "step": 2002
+    },
+    {
+      "epoch": 2.492843808338519,
+      "grad_norm": 3.389965057373047,
+      "learning_rate": 9.732472324723247e-07,
+      "loss": 1.2777,
+      "step": 2003
+    },
+    {
+      "epoch": 2.4940883634100808,
+      "grad_norm": 3.370044231414795,
+      "learning_rate": 9.709409594095941e-07,
+      "loss": 1.3407,
+      "step": 2004
+    },
+    {
+      "epoch": 2.4953329184816426,
+      "grad_norm": 3.4178175926208496,
+      "learning_rate": 9.686346863468636e-07,
+      "loss": 1.3419,
+      "step": 2005
+    },
+    {
+      "epoch": 2.496577473553205,
+      "grad_norm": 3.4306576251983643,
+      "learning_rate": 9.66328413284133e-07,
+      "loss": 1.3029,
+      "step": 2006
+    },
+    {
+      "epoch": 2.4978220286247668,
+      "grad_norm": 3.4822888374328613,
+      "learning_rate": 9.640221402214022e-07,
+      "loss": 1.3512,
+      "step": 2007
+    },
+    {
+      "epoch": 2.4990665836963286,
+      "grad_norm": 3.2686867713928223,
+      "learning_rate": 9.617158671586716e-07,
+      "loss": 1.3366,
+      "step": 2008
+    },
+    {
+      "epoch": 2.5003111387678905,
+      "grad_norm": 3.413224220275879,
+      "learning_rate": 9.59409594095941e-07,
+      "loss": 1.324,
+      "step": 2009
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "grad_norm": 3.180180311203003,
+      "learning_rate": 9.571033210332105e-07,
+      "loss": 1.3054,
+      "step": 2010
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "eval_loss": 1.6145573854446411,
+      "eval_runtime": 42.6598,
+      "eval_samples_per_second": 23.441,
+      "eval_steps_per_second": 0.985,
+      "step": 2010
+    },
+    {
+      "epoch": 2.502800248911014,
+      "grad_norm": 3.4968016147613525,
+      "learning_rate": 9.547970479704797e-07,
+      "loss": 1.3782,
+      "step": 2011
+    },
+    {
+      "epoch": 2.504044803982576,
+      "grad_norm": 3.270414113998413,
+      "learning_rate": 9.524907749077492e-07,
+      "loss": 1.3389,
+      "step": 2012
+    },
+    {
+      "epoch": 2.5052893590541383,
+      "grad_norm": 3.3850443363189697,
+      "learning_rate": 9.501845018450185e-07,
+      "loss": 1.3335,
+      "step": 2013
+    },
+    {
+      "epoch": 2.5065339141257,
+      "grad_norm": 3.3836278915405273,
+      "learning_rate": 9.478782287822879e-07,
+      "loss": 1.3343,
+      "step": 2014
+    },
+    {
+      "epoch": 2.507778469197262,
+      "grad_norm": 3.4950475692749023,
+      "learning_rate": 9.455719557195573e-07,
+      "loss": 1.337,
+      "step": 2015
+    },
+    {
+      "epoch": 2.509023024268824,
+      "grad_norm": 3.368262529373169,
+      "learning_rate": 9.432656826568267e-07,
+      "loss": 1.32,
+      "step": 2016
+    },
+    {
+      "epoch": 2.5102675793403857,
+      "grad_norm": 3.4199533462524414,
+      "learning_rate": 9.40959409594096e-07,
+      "loss": 1.3754,
+      "step": 2017
+    },
+    {
+      "epoch": 2.511512134411948,
+      "grad_norm": 3.3372180461883545,
+      "learning_rate": 9.386531365313654e-07,
+      "loss": 1.3319,
+      "step": 2018
+    },
+    {
+      "epoch": 2.51275668948351,
+      "grad_norm": 3.3614137172698975,
+      "learning_rate": 9.363468634686347e-07,
+      "loss": 1.383,
+      "step": 2019
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "grad_norm": 3.4617390632629395,
+      "learning_rate": 9.340405904059042e-07,
+      "loss": 1.3628,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "eval_loss": 1.6153606176376343,
+      "eval_runtime": 46.8889,
+      "eval_samples_per_second": 21.327,
+      "eval_steps_per_second": 0.896,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5152457996266335,
+      "grad_norm": 3.3711016178131104,
+      "learning_rate": 9.317343173431735e-07,
+      "loss": 1.3449,
+      "step": 2021
+    },
+    {
+      "epoch": 2.5164903546981954,
+      "grad_norm": 3.2858471870422363,
+      "learning_rate": 9.294280442804429e-07,
+      "loss": 1.348,
+      "step": 2022
+    },
+    {
+      "epoch": 2.5177349097697572,
+      "grad_norm": 3.445166826248169,
+      "learning_rate": 9.271217712177122e-07,
+      "loss": 1.3182,
+      "step": 2023
+    },
+    {
+      "epoch": 2.518979464841319,
+      "grad_norm": 3.417206287384033,
+      "learning_rate": 9.248154981549816e-07,
+      "loss": 1.3241,
+      "step": 2024
+    },
+    {
+      "epoch": 2.520224019912881,
+      "grad_norm": 3.333965539932251,
+      "learning_rate": 9.22509225092251e-07,
+      "loss": 1.3214,
+      "step": 2025
+    },
+    {
+      "epoch": 2.521468574984443,
+      "grad_norm": 3.4090497493743896,
+      "learning_rate": 9.202029520295204e-07,
+      "loss": 1.3566,
+      "step": 2026
+    },
+    {
+      "epoch": 2.522713130056005,
+      "grad_norm": 3.312790870666504,
+      "learning_rate": 9.178966789667898e-07,
+      "loss": 1.3181,
+      "step": 2027
+    },
+    {
+      "epoch": 2.523957685127567,
+      "grad_norm": 3.254122734069824,
+      "learning_rate": 9.155904059040591e-07,
+      "loss": 1.3079,
+      "step": 2028
+    },
+    {
+      "epoch": 2.5252022401991288,
+      "grad_norm": 3.5144238471984863,
+      "learning_rate": 9.132841328413284e-07,
+      "loss": 1.3586,
+      "step": 2029
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "grad_norm": 3.6083173751831055,
+      "learning_rate": 9.109778597785979e-07,
+      "loss": 1.3833,
+      "step": 2030
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "eval_loss": 1.6153796911239624,
+      "eval_runtime": 52.1312,
+      "eval_samples_per_second": 19.182,
+      "eval_steps_per_second": 0.806,
+      "step": 2030
+    },
+    {
+      "epoch": 2.527691350342253,
+      "grad_norm": 3.489710569381714,
+      "learning_rate": 9.086715867158673e-07,
+      "loss": 1.3466,
+      "step": 2031
+    },
+    {
+      "epoch": 2.5289359054138147,
+      "grad_norm": 3.4363298416137695,
+      "learning_rate": 9.063653136531365e-07,
+      "loss": 1.3461,
+      "step": 2032
+    },
+    {
+      "epoch": 2.5301804604853766,
+      "grad_norm": 3.627476930618286,
+      "learning_rate": 9.040590405904059e-07,
+      "loss": 1.3655,
+      "step": 2033
+    },
+    {
+      "epoch": 2.5314250155569384,
+      "grad_norm": 3.2995362281799316,
+      "learning_rate": 9.017527675276753e-07,
+      "loss": 1.297,
+      "step": 2034
+    },
+    {
+      "epoch": 2.5326695706285003,
+      "grad_norm": 3.4116721153259277,
+      "learning_rate": 8.994464944649448e-07,
+      "loss": 1.3319,
+      "step": 2035
+    },
+    {
+      "epoch": 2.533914125700062,
+      "grad_norm": 3.4598546028137207,
+      "learning_rate": 8.971402214022142e-07,
+      "loss": 1.3646,
+      "step": 2036
+    },
+    {
+      "epoch": 2.535158680771624,
+      "grad_norm": 3.4111695289611816,
+      "learning_rate": 8.948339483394834e-07,
+      "loss": 1.322,
+      "step": 2037
+    },
+    {
+      "epoch": 2.536403235843186,
+      "grad_norm": 3.371891736984253,
+      "learning_rate": 8.925276752767528e-07,
+      "loss": 1.362,
+      "step": 2038
+    },
+    {
+      "epoch": 2.537647790914748,
+      "grad_norm": 3.4267642498016357,
+      "learning_rate": 8.902214022140222e-07,
+      "loss": 1.3829,
+      "step": 2039
+    },
+    {
+      "epoch": 2.53889234598631,
+      "grad_norm": 3.2384777069091797,
+      "learning_rate": 8.879151291512917e-07,
+      "loss": 1.2771,
+      "step": 2040
+    },
+    {
+      "epoch": 2.53889234598631,
+      "eval_loss": 1.6134564876556396,
+      "eval_runtime": 49.4544,
+      "eval_samples_per_second": 20.221,
+      "eval_steps_per_second": 0.849,
+      "step": 2040
+    },
+    {
+      "epoch": 2.540136901057872,
+      "grad_norm": 3.3554086685180664,
+      "learning_rate": 8.856088560885609e-07,
+      "loss": 1.327,
+      "step": 2041
+    },
+    {
+      "epoch": 2.5413814561294337,
+      "grad_norm": 3.297962188720703,
+      "learning_rate": 8.833025830258303e-07,
+      "loss": 1.3257,
+      "step": 2042
+    },
+    {
+      "epoch": 2.5426260112009955,
+      "grad_norm": 3.4998221397399902,
+      "learning_rate": 8.809963099630997e-07,
+      "loss": 1.3782,
+      "step": 2043
+    },
+    {
+      "epoch": 2.543870566272558,
+      "grad_norm": 3.3661816120147705,
+      "learning_rate": 8.786900369003691e-07,
+      "loss": 1.3338,
+      "step": 2044
+    },
+    {
+      "epoch": 2.5451151213441197,
+      "grad_norm": 3.7328436374664307,
+      "learning_rate": 8.763837638376385e-07,
+      "loss": 1.4149,
+      "step": 2045
+    },
+    {
+      "epoch": 2.5463596764156815,
+      "grad_norm": 3.621159315109253,
+      "learning_rate": 8.740774907749078e-07,
+      "loss": 1.3104,
+      "step": 2046
+    },
+    {
+      "epoch": 2.5476042314872434,
+      "grad_norm": 3.5403215885162354,
+      "learning_rate": 8.717712177121772e-07,
+      "loss": 1.3278,
+      "step": 2047
+    },
+    {
+      "epoch": 2.548848786558805,
+      "grad_norm": 3.3124406337738037,
+      "learning_rate": 8.694649446494466e-07,
+      "loss": 1.3313,
+      "step": 2048
+    },
+    {
+      "epoch": 2.550093341630367,
+      "grad_norm": 3.4480574131011963,
+      "learning_rate": 8.671586715867159e-07,
+      "loss": 1.3225,
+      "step": 2049
+    },
+    {
+      "epoch": 2.551337896701929,
+      "grad_norm": 3.372326135635376,
+      "learning_rate": 8.648523985239853e-07,
+      "loss": 1.3377,
+      "step": 2050
+    },
+    {
+      "epoch": 2.551337896701929,
+      "eval_loss": 1.6056281328201294,
+      "eval_runtime": 52.0326,
+      "eval_samples_per_second": 19.219,
+      "eval_steps_per_second": 0.807,
+      "step": 2050
+    },
+    {
+      "epoch": 2.5525824517734907,
+      "grad_norm": 3.4119911193847656,
+      "learning_rate": 8.625461254612547e-07,
+      "loss": 1.3462,
+      "step": 2051
+    },
+    {
+      "epoch": 2.553827006845053,
+      "grad_norm": 3.5083751678466797,
+      "learning_rate": 8.602398523985241e-07,
+      "loss": 1.3632,
+      "step": 2052
+    },
+    {
+      "epoch": 2.555071561916615,
+      "grad_norm": 3.279911518096924,
+      "learning_rate": 8.579335793357934e-07,
+      "loss": 1.3326,
+      "step": 2053
+    },
+    {
+      "epoch": 2.5563161169881767,
+      "grad_norm": 3.2245192527770996,
+      "learning_rate": 8.556273062730628e-07,
+      "loss": 1.3033,
+      "step": 2054
+    },
+    {
+      "epoch": 2.5575606720597386,
+      "grad_norm": 3.3703253269195557,
+      "learning_rate": 8.533210332103322e-07,
+      "loss": 1.3219,
+      "step": 2055
+    },
+    {
+      "epoch": 2.5588052271313004,
+      "grad_norm": 3.3931314945220947,
+      "learning_rate": 8.510147601476016e-07,
+      "loss": 1.2992,
+      "step": 2056
+    },
+    {
+      "epoch": 2.5600497822028627,
+      "grad_norm": 3.5495870113372803,
+      "learning_rate": 8.487084870848709e-07,
+      "loss": 1.3872,
+      "step": 2057
+    },
+    {
+      "epoch": 2.5612943372744246,
+      "grad_norm": 3.476536273956299,
+      "learning_rate": 8.464022140221403e-07,
+      "loss": 1.3609,
+      "step": 2058
+    },
+    {
+      "epoch": 2.5625388923459864,
+      "grad_norm": 3.4503397941589355,
+      "learning_rate": 8.440959409594096e-07,
+      "loss": 1.339,
+      "step": 2059
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "grad_norm": 3.3848440647125244,
+      "learning_rate": 8.417896678966791e-07,
+      "loss": 1.3369,
+      "step": 2060
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "eval_loss": 1.61435866355896,
+      "eval_runtime": 48.4317,
+      "eval_samples_per_second": 20.648,
+      "eval_steps_per_second": 0.867,
+      "step": 2060
+    },
+    {
+      "epoch": 2.56502800248911,
+      "grad_norm": 3.474571466445923,
+      "learning_rate": 8.394833948339484e-07,
+      "loss": 1.379,
+      "step": 2061
+    },
+    {
+      "epoch": 2.566272557560672,
+      "grad_norm": 3.487356185913086,
+      "learning_rate": 8.371771217712178e-07,
+      "loss": 1.365,
+      "step": 2062
+    },
+    {
+      "epoch": 2.567517112632234,
+      "grad_norm": 3.5649149417877197,
+      "learning_rate": 8.348708487084871e-07,
+      "loss": 1.3619,
+      "step": 2063
+    },
+    {
+      "epoch": 2.5687616677037957,
+      "grad_norm": 3.572577714920044,
+      "learning_rate": 8.325645756457565e-07,
+      "loss": 1.3877,
+      "step": 2064
+    },
+    {
+      "epoch": 2.570006222775358,
+      "grad_norm": 3.3802125453948975,
+      "learning_rate": 8.302583025830259e-07,
+      "loss": 1.3323,
+      "step": 2065
+    },
+    {
+      "epoch": 2.57125077784692,
+      "grad_norm": 3.317291498184204,
+      "learning_rate": 8.279520295202953e-07,
+      "loss": 1.3532,
+      "step": 2066
+    },
+    {
+      "epoch": 2.5724953329184816,
+      "grad_norm": 3.485429525375366,
+      "learning_rate": 8.256457564575647e-07,
+      "loss": 1.3561,
+      "step": 2067
+    },
+    {
+      "epoch": 2.5737398879900435,
+      "grad_norm": 3.4251649379730225,
+      "learning_rate": 8.233394833948339e-07,
+      "loss": 1.3494,
+      "step": 2068
+    },
+    {
+      "epoch": 2.5749844430616053,
+      "grad_norm": 3.3372538089752197,
+      "learning_rate": 8.210332103321033e-07,
+      "loss": 1.3116,
+      "step": 2069
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "grad_norm": 3.3535027503967285,
+      "learning_rate": 8.187269372693728e-07,
+      "loss": 1.3541,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "eval_loss": 1.6068458557128906,
+      "eval_runtime": 51.6237,
+      "eval_samples_per_second": 19.371,
+      "eval_steps_per_second": 0.814,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5774735532047295,
+      "grad_norm": 3.4335787296295166,
+      "learning_rate": 8.164206642066422e-07,
+      "loss": 1.3884,
+      "step": 2071
+    },
+    {
+      "epoch": 2.5787181082762913,
+      "grad_norm": 3.436685800552368,
+      "learning_rate": 8.141143911439114e-07,
+      "loss": 1.3508,
+      "step": 2072
+    },
+    {
+      "epoch": 2.579962663347853,
+      "grad_norm": 3.333679676055908,
+      "learning_rate": 8.118081180811808e-07,
+      "loss": 1.3485,
+      "step": 2073
+    },
+    {
+      "epoch": 2.581207218419415,
+      "grad_norm": 3.4076309204101562,
+      "learning_rate": 8.095018450184502e-07,
+      "loss": 1.3219,
+      "step": 2074
+    },
+    {
+      "epoch": 2.582451773490977,
+      "grad_norm": 3.3851561546325684,
+      "learning_rate": 8.071955719557197e-07,
+      "loss": 1.3365,
+      "step": 2075
+    },
+    {
+      "epoch": 2.5836963285625387,
+      "grad_norm": 3.3714189529418945,
+      "learning_rate": 8.048892988929891e-07,
+      "loss": 1.3448,
+      "step": 2076
+    },
+    {
+      "epoch": 2.5849408836341006,
+      "grad_norm": 3.32585072517395,
+      "learning_rate": 8.025830258302583e-07,
+      "loss": 1.338,
+      "step": 2077
+    },
+    {
+      "epoch": 2.586185438705663,
+      "grad_norm": 3.4017553329467773,
+      "learning_rate": 8.002767527675277e-07,
+      "loss": 1.3597,
+      "step": 2078
+    },
+    {
+      "epoch": 2.5874299937772247,
+      "grad_norm": 3.334178924560547,
+      "learning_rate": 7.979704797047971e-07,
+      "loss": 1.2981,
+      "step": 2079
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "grad_norm": 3.443598985671997,
+      "learning_rate": 7.956642066420666e-07,
+      "loss": 1.3188,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "eval_loss": 1.6108036041259766,
+      "eval_runtime": 50.3752,
+      "eval_samples_per_second": 19.851,
+      "eval_steps_per_second": 0.834,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5899191039203484,
+      "grad_norm": 3.460387706756592,
+      "learning_rate": 7.933579335793358e-07,
+      "loss": 1.3794,
+      "step": 2081
+    },
+    {
+      "epoch": 2.5911636589919103,
+      "grad_norm": 3.370560646057129,
+      "learning_rate": 7.910516605166052e-07,
+      "loss": 1.3068,
+      "step": 2082
+    },
+    {
+      "epoch": 2.5924082140634725,
+      "grad_norm": 3.55534291267395,
+      "learning_rate": 7.887453874538746e-07,
+      "loss": 1.3581,
+      "step": 2083
+    },
+    {
+      "epoch": 2.5936527691350344,
+      "grad_norm": 3.322735548019409,
+      "learning_rate": 7.86439114391144e-07,
+      "loss": 1.3399,
+      "step": 2084
+    },
+    {
+      "epoch": 2.5948973242065962,
+      "grad_norm": 3.5256693363189697,
+      "learning_rate": 7.841328413284134e-07,
+      "loss": 1.377,
+      "step": 2085
+    },
+    {
+      "epoch": 2.596141879278158,
+      "grad_norm": 3.4358303546905518,
+      "learning_rate": 7.818265682656827e-07,
+      "loss": 1.3486,
+      "step": 2086
+    },
+    {
+      "epoch": 2.59738643434972,
+      "grad_norm": 3.401855230331421,
+      "learning_rate": 7.795202952029521e-07,
+      "loss": 1.3013,
+      "step": 2087
+    },
+    {
+      "epoch": 2.598630989421282,
+      "grad_norm": 3.414670467376709,
+      "learning_rate": 7.772140221402215e-07,
+      "loss": 1.3342,
+      "step": 2088
+    },
+    {
+      "epoch": 2.5998755444928436,
+      "grad_norm": 3.3348028659820557,
+      "learning_rate": 7.749077490774908e-07,
+      "loss": 1.3594,
+      "step": 2089
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "grad_norm": 3.4095754623413086,
+      "learning_rate": 7.726014760147602e-07,
+      "loss": 1.3739,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "eval_loss": 1.6106606721878052,
+      "eval_runtime": 49.3878,
+      "eval_samples_per_second": 20.248,
+      "eval_steps_per_second": 0.85,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6023646546359678,
+      "grad_norm": 3.247753143310547,
+      "learning_rate": 7.702952029520296e-07,
+      "loss": 1.3026,
+      "step": 2091
+    },
+    {
+      "epoch": 2.6036092097075296,
+      "grad_norm": 3.3973324298858643,
+      "learning_rate": 7.67988929889299e-07,
+      "loss": 1.3381,
+      "step": 2092
+    },
+    {
+      "epoch": 2.6048537647790915,
+      "grad_norm": 3.4262280464172363,
+      "learning_rate": 7.656826568265683e-07,
+      "loss": 1.3595,
+      "step": 2093
+    },
+    {
+      "epoch": 2.6060983198506533,
+      "grad_norm": 3.401784658432007,
+      "learning_rate": 7.633763837638377e-07,
+      "loss": 1.3339,
+      "step": 2094
+    },
+    {
+      "epoch": 2.607342874922215,
+      "grad_norm": 3.3231465816497803,
+      "learning_rate": 7.610701107011071e-07,
+      "loss": 1.2529,
+      "step": 2095
+    },
+    {
+      "epoch": 2.6085874299937775,
+      "grad_norm": 3.412613868713379,
+      "learning_rate": 7.587638376383765e-07,
+      "loss": 1.3341,
+      "step": 2096
+    },
+    {
+      "epoch": 2.6098319850653393,
+      "grad_norm": 3.456622362136841,
+      "learning_rate": 7.564575645756458e-07,
+      "loss": 1.3545,
+      "step": 2097
+    },
+    {
+      "epoch": 2.611076540136901,
+      "grad_norm": 3.488546133041382,
+      "learning_rate": 7.541512915129152e-07,
+      "loss": 1.3405,
+      "step": 2098
+    },
+    {
+      "epoch": 2.612321095208463,
+      "grad_norm": 3.4436495304107666,
+      "learning_rate": 7.518450184501845e-07,
+      "loss": 1.3242,
+      "step": 2099
+    },
+    {
+      "epoch": 2.613565650280025,
+      "grad_norm": 3.382150411605835,
+      "learning_rate": 7.49538745387454e-07,
+      "loss": 1.301,
+      "step": 2100
+    },
+    {
+      "epoch": 2.613565650280025,
+      "eval_loss": 1.6055735349655151,
+      "eval_runtime": 48.001,
+      "eval_samples_per_second": 20.833,
+      "eval_steps_per_second": 0.875,
+      "step": 2100
+    },
+    {
+      "epoch": 2.6148102053515867,
+      "grad_norm": 3.478010892868042,
+      "learning_rate": 7.472324723247233e-07,
+      "loss": 1.3565,
+      "step": 2101
+    },
+    {
+      "epoch": 2.6160547604231486,
+      "grad_norm": 3.4550209045410156,
+      "learning_rate": 7.449261992619927e-07,
+      "loss": 1.3732,
+      "step": 2102
+    },
+    {
+      "epoch": 2.6172993154947104,
+      "grad_norm": 3.3357841968536377,
+      "learning_rate": 7.42619926199262e-07,
+      "loss": 1.3072,
+      "step": 2103
+    },
+    {
+      "epoch": 2.6185438705662727,
+      "grad_norm": 3.4329988956451416,
+      "learning_rate": 7.403136531365313e-07,
+      "loss": 1.3353,
+      "step": 2104
+    },
+    {
+      "epoch": 2.6197884256378345,
+      "grad_norm": 3.390272378921509,
+      "learning_rate": 7.380073800738008e-07,
+      "loss": 1.3537,
+      "step": 2105
+    },
+    {
+      "epoch": 2.6210329807093964,
+      "grad_norm": 3.4377708435058594,
+      "learning_rate": 7.357011070110702e-07,
+      "loss": 1.357,
+      "step": 2106
+    },
+    {
+      "epoch": 2.6222775357809582,
+      "grad_norm": 3.383354425430298,
+      "learning_rate": 7.333948339483396e-07,
+      "loss": 1.3663,
+      "step": 2107
+    },
+    {
+      "epoch": 2.62352209085252,
+      "grad_norm": 3.364445209503174,
+      "learning_rate": 7.310885608856088e-07,
+      "loss": 1.3515,
+      "step": 2108
+    },
+    {
+      "epoch": 2.6247666459240824,
+      "grad_norm": 3.357224702835083,
+      "learning_rate": 7.287822878228782e-07,
+      "loss": 1.3436,
+      "step": 2109
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "grad_norm": 3.340733289718628,
+      "learning_rate": 7.264760147601477e-07,
+      "loss": 1.3481,
+      "step": 2110
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "eval_loss": 1.609564185142517,
+      "eval_runtime": 43.5251,
+      "eval_samples_per_second": 22.975,
+      "eval_steps_per_second": 0.965,
+      "step": 2110
+    },
+    {
+      "epoch": 2.627255756067206,
+      "grad_norm": 3.307832717895508,
+      "learning_rate": 7.241697416974171e-07,
+      "loss": 1.322,
+      "step": 2111
+    },
+    {
+      "epoch": 2.628500311138768,
+      "grad_norm": 3.3134219646453857,
+      "learning_rate": 7.218634686346863e-07,
+      "loss": 1.2888,
+      "step": 2112
+    },
+    {
+      "epoch": 2.6297448662103298,
+      "grad_norm": 3.5507848262786865,
+      "learning_rate": 7.195571955719557e-07,
+      "loss": 1.3986,
+      "step": 2113
+    },
+    {
+      "epoch": 2.6309894212818916,
+      "grad_norm": 3.26234769821167,
+      "learning_rate": 7.172509225092251e-07,
+      "loss": 1.3037,
+      "step": 2114
+    },
+    {
+      "epoch": 2.6322339763534535,
+      "grad_norm": 3.4239728450775146,
+      "learning_rate": 7.149446494464946e-07,
+      "loss": 1.326,
+      "step": 2115
+    },
+    {
+      "epoch": 2.6334785314250153,
+      "grad_norm": 3.470409870147705,
+      "learning_rate": 7.12638376383764e-07,
+      "loss": 1.3435,
+      "step": 2116
+    },
+    {
+      "epoch": 2.6347230864965776,
+      "grad_norm": 3.4207205772399902,
+      "learning_rate": 7.103321033210332e-07,
+      "loss": 1.3519,
+      "step": 2117
+    },
+    {
+      "epoch": 2.6359676415681395,
+      "grad_norm": 3.3702151775360107,
+      "learning_rate": 7.080258302583026e-07,
+      "loss": 1.3081,
+      "step": 2118
+    },
+    {
+      "epoch": 2.6372121966397013,
+      "grad_norm": 3.406705379486084,
+      "learning_rate": 7.05719557195572e-07,
+      "loss": 1.3698,
+      "step": 2119
+    },
+    {
+      "epoch": 2.638456751711263,
+      "grad_norm": 3.5395123958587646,
+      "learning_rate": 7.034132841328415e-07,
+      "loss": 1.3543,
+      "step": 2120
+    },
+    {
+      "epoch": 2.638456751711263,
+      "eval_loss": 1.6064285039901733,
+      "eval_runtime": 47.1124,
+      "eval_samples_per_second": 21.226,
+      "eval_steps_per_second": 0.891,
+      "step": 2120
+    },
+    {
+      "epoch": 2.639701306782825,
+      "grad_norm": 3.3317949771881104,
+      "learning_rate": 7.011070110701107e-07,
+      "loss": 1.3401,
+      "step": 2121
+    },
+    {
+      "epoch": 2.6409458618543873,
+      "grad_norm": 3.3046767711639404,
+      "learning_rate": 6.988007380073801e-07,
+      "loss": 1.355,
+      "step": 2122
+    },
+    {
+      "epoch": 2.642190416925949,
+      "grad_norm": 3.419886350631714,
+      "learning_rate": 6.964944649446495e-07,
+      "loss": 1.2955,
+      "step": 2123
+    },
+    {
+      "epoch": 2.643434971997511,
+      "grad_norm": 3.3755574226379395,
+      "learning_rate": 6.94188191881919e-07,
+      "loss": 1.3093,
+      "step": 2124
+    },
+    {
+      "epoch": 2.644679527069073,
+      "grad_norm": 3.490931510925293,
+      "learning_rate": 6.918819188191883e-07,
+      "loss": 1.3994,
+      "step": 2125
+    },
+    {
+      "epoch": 2.6459240821406347,
+      "grad_norm": 3.48911452293396,
+      "learning_rate": 6.895756457564576e-07,
+      "loss": 1.3449,
+      "step": 2126
+    },
+    {
+      "epoch": 2.6471686372121965,
+      "grad_norm": 3.5786304473876953,
+      "learning_rate": 6.87269372693727e-07,
+      "loss": 1.3569,
+      "step": 2127
+    },
+    {
+      "epoch": 2.6484131922837584,
+      "grad_norm": 3.500648260116577,
+      "learning_rate": 6.849630996309964e-07,
+      "loss": 1.349,
+      "step": 2128
+    },
+    {
+      "epoch": 2.6496577473553202,
+      "grad_norm": 3.644001007080078,
+      "learning_rate": 6.826568265682657e-07,
+      "loss": 1.3534,
+      "step": 2129
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "grad_norm": 3.3233609199523926,
+      "learning_rate": 6.803505535055351e-07,
+      "loss": 1.303,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "eval_loss": 1.6085610389709473,
+      "eval_runtime": 51.443,
+      "eval_samples_per_second": 19.439,
+      "eval_steps_per_second": 0.816,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6521468574984444,
+      "grad_norm": 3.413889169692993,
+      "learning_rate": 6.780442804428045e-07,
+      "loss": 1.335,
+      "step": 2131
+    },
+    {
+      "epoch": 2.653391412570006,
+      "grad_norm": 3.417112112045288,
+      "learning_rate": 6.757380073800739e-07,
+      "loss": 1.3768,
+      "step": 2132
+    },
+    {
+      "epoch": 2.654635967641568,
+      "grad_norm": 3.4491703510284424,
+      "learning_rate": 6.734317343173432e-07,
+      "loss": 1.3493,
+      "step": 2133
+    },
+    {
+      "epoch": 2.65588052271313,
+      "grad_norm": 3.3965368270874023,
+      "learning_rate": 6.711254612546127e-07,
+      "loss": 1.3328,
+      "step": 2134
+    },
+    {
+      "epoch": 2.657125077784692,
+      "grad_norm": 3.426530361175537,
+      "learning_rate": 6.68819188191882e-07,
+      "loss": 1.3548,
+      "step": 2135
+    },
+    {
+      "epoch": 2.658369632856254,
+      "grad_norm": 3.386624813079834,
+      "learning_rate": 6.665129151291514e-07,
+      "loss": 1.3523,
+      "step": 2136
+    },
+    {
+      "epoch": 2.659614187927816,
+      "grad_norm": 3.383504867553711,
+      "learning_rate": 6.642066420664207e-07,
+      "loss": 1.3356,
+      "step": 2137
+    },
+    {
+      "epoch": 2.6608587429993777,
+      "grad_norm": 3.3986196517944336,
+      "learning_rate": 6.619003690036901e-07,
+      "loss": 1.3605,
+      "step": 2138
+    },
+    {
+      "epoch": 2.6621032980709396,
+      "grad_norm": 3.4832820892333984,
+      "learning_rate": 6.595940959409594e-07,
+      "loss": 1.395,
+      "step": 2139
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "grad_norm": 3.498077154159546,
+      "learning_rate": 6.572878228782288e-07,
+      "loss": 1.347,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "eval_loss": 1.6076886653900146,
+      "eval_runtime": 47.7587,
+      "eval_samples_per_second": 20.939,
+      "eval_steps_per_second": 0.879,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6645924082140633,
+      "grad_norm": 3.6382765769958496,
+      "learning_rate": 6.549815498154982e-07,
+      "loss": 1.4288,
+      "step": 2141
+    },
+    {
+      "epoch": 2.665836963285625,
+      "grad_norm": 3.434046983718872,
+      "learning_rate": 6.526752767527676e-07,
+      "loss": 1.3258,
+      "step": 2142
+    },
+    {
+      "epoch": 2.6670815183571874,
+      "grad_norm": 3.381760358810425,
+      "learning_rate": 6.503690036900369e-07,
+      "loss": 1.3567,
+      "step": 2143
+    },
+    {
+      "epoch": 2.6683260734287493,
+      "grad_norm": 3.450314521789551,
+      "learning_rate": 6.480627306273062e-07,
+      "loss": 1.3504,
+      "step": 2144
+    },
+    {
+      "epoch": 2.669570628500311,
+      "grad_norm": 3.337059736251831,
+      "learning_rate": 6.457564575645757e-07,
+      "loss": 1.3664,
+      "step": 2145
+    },
+    {
+      "epoch": 2.670815183571873,
+      "grad_norm": 3.415597438812256,
+      "learning_rate": 6.434501845018451e-07,
+      "loss": 1.369,
+      "step": 2146
+    },
+    {
+      "epoch": 2.672059738643435,
+      "grad_norm": 3.331700325012207,
+      "learning_rate": 6.411439114391145e-07,
+      "loss": 1.3056,
+      "step": 2147
+    },
+    {
+      "epoch": 2.673304293714997,
+      "grad_norm": 3.439768075942993,
+      "learning_rate": 6.388376383763837e-07,
+      "loss": 1.3591,
+      "step": 2148
+    },
+    {
+      "epoch": 2.674548848786559,
+      "grad_norm": 3.4256532192230225,
+      "learning_rate": 6.365313653136531e-07,
+      "loss": 1.3581,
+      "step": 2149
+    },
+    {
+      "epoch": 2.675793403858121,
+      "grad_norm": 3.337472438812256,
+      "learning_rate": 6.342250922509226e-07,
+      "loss": 1.358,
+      "step": 2150
+    },
+    {
+      "epoch": 2.675793403858121,
+      "eval_loss": 1.6052860021591187,
+      "eval_runtime": 51.6432,
+      "eval_samples_per_second": 19.364,
+      "eval_steps_per_second": 0.813,
+      "step": 2150
+    },
+    {
+      "epoch": 2.6770379589296827,
+      "grad_norm": 3.458285331726074,
+      "learning_rate": 6.31918819188192e-07,
+      "loss": 1.346,
+      "step": 2151
+    },
+    {
+      "epoch": 2.6782825140012445,
+      "grad_norm": 3.3939685821533203,
+      "learning_rate": 6.296125461254612e-07,
+      "loss": 1.3634,
+      "step": 2152
+    },
+    {
+      "epoch": 2.6795270690728064,
+      "grad_norm": 3.4758291244506836,
+      "learning_rate": 6.273062730627306e-07,
+      "loss": 1.3913,
+      "step": 2153
+    },
+    {
+      "epoch": 2.680771624144368,
+      "grad_norm": 3.316906690597534,
+      "learning_rate": 6.25e-07,
+      "loss": 1.2944,
+      "step": 2154
+    },
+    {
+      "epoch": 2.68201617921593,
+      "grad_norm": 3.357058048248291,
+      "learning_rate": 6.226937269372695e-07,
+      "loss": 1.321,
+      "step": 2155
+    },
+    {
+      "epoch": 2.6832607342874923,
+      "grad_norm": 3.4547276496887207,
+      "learning_rate": 6.203874538745388e-07,
+      "loss": 1.3407,
+      "step": 2156
+    },
+    {
+      "epoch": 2.684505289359054,
+      "grad_norm": 3.3258347511291504,
+      "learning_rate": 6.180811808118082e-07,
+      "loss": 1.3526,
+      "step": 2157
+    },
+    {
+      "epoch": 2.685749844430616,
+      "grad_norm": 3.418070077896118,
+      "learning_rate": 6.157749077490775e-07,
+      "loss": 1.3764,
+      "step": 2158
+    },
+    {
+      "epoch": 2.686994399502178,
+      "grad_norm": 3.2903330326080322,
+      "learning_rate": 6.13468634686347e-07,
+      "loss": 1.3144,
+      "step": 2159
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "grad_norm": 3.340200901031494,
+      "learning_rate": 6.111623616236163e-07,
+      "loss": 1.3399,
+      "step": 2160
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "eval_loss": 1.6079058647155762,
+      "eval_runtime": 52.2924,
+      "eval_samples_per_second": 19.123,
+      "eval_steps_per_second": 0.803,
+      "step": 2160
+    },
+    {
+      "epoch": 2.689483509645302,
+      "grad_norm": 3.4739818572998047,
+      "learning_rate": 6.088560885608857e-07,
+      "loss": 1.333,
+      "step": 2161
+    },
+    {
+      "epoch": 2.690728064716864,
+      "grad_norm": 3.5063717365264893,
+      "learning_rate": 6.06549815498155e-07,
+      "loss": 1.3089,
+      "step": 2162
+    },
+    {
+      "epoch": 2.6919726197884257,
+      "grad_norm": 3.34189510345459,
+      "learning_rate": 6.042435424354244e-07,
+      "loss": 1.3173,
+      "step": 2163
+    },
+    {
+      "epoch": 2.6932171748599876,
+      "grad_norm": 3.385422706604004,
+      "learning_rate": 6.019372693726938e-07,
+      "loss": 1.3605,
+      "step": 2164
+    },
+    {
+      "epoch": 2.6944617299315494,
+      "grad_norm": 3.3808515071868896,
+      "learning_rate": 5.996309963099632e-07,
+      "loss": 1.3341,
+      "step": 2165
+    },
+    {
+      "epoch": 2.6957062850031113,
+      "grad_norm": 3.546069383621216,
+      "learning_rate": 5.973247232472326e-07,
+      "loss": 1.3706,
+      "step": 2166
+    },
+    {
+      "epoch": 2.696950840074673,
+      "grad_norm": 3.3705339431762695,
+      "learning_rate": 5.950184501845019e-07,
+      "loss": 1.3226,
+      "step": 2167
+    },
+    {
+      "epoch": 2.698195395146235,
+      "grad_norm": 3.4087791442871094,
+      "learning_rate": 5.927121771217713e-07,
+      "loss": 1.3285,
+      "step": 2168
+    },
+    {
+      "epoch": 2.6994399502177973,
+      "grad_norm": 3.2774477005004883,
+      "learning_rate": 5.904059040590407e-07,
+      "loss": 1.3128,
+      "step": 2169
+    },
+    {
+      "epoch": 2.700684505289359,
+      "grad_norm": 3.3653030395507812,
+      "learning_rate": 5.880996309963101e-07,
+      "loss": 1.3395,
+      "step": 2170
+    },
+    {
+      "epoch": 2.700684505289359,
+      "eval_loss": 1.608220100402832,
+      "eval_runtime": 53.3978,
+      "eval_samples_per_second": 18.727,
+      "eval_steps_per_second": 0.787,
+      "step": 2170
+    },
+    {
+      "epoch": 2.701929060360921,
+      "grad_norm": 3.394524335861206,
+      "learning_rate": 5.857933579335794e-07,
+      "loss": 1.2853,
+      "step": 2171
+    },
+    {
+      "epoch": 2.703173615432483,
+      "grad_norm": 3.3898823261260986,
+      "learning_rate": 5.834870848708487e-07,
+      "loss": 1.3494,
+      "step": 2172
+    },
+    {
+      "epoch": 2.7044181705040446,
+      "grad_norm": 3.394688367843628,
+      "learning_rate": 5.811808118081181e-07,
+      "loss": 1.3129,
+      "step": 2173
+    },
+    {
+      "epoch": 2.705662725575607,
+      "grad_norm": 3.4164483547210693,
+      "learning_rate": 5.788745387453875e-07,
+      "loss": 1.368,
+      "step": 2174
+    },
+    {
+      "epoch": 2.706907280647169,
+      "grad_norm": 3.2976245880126953,
+      "learning_rate": 5.765682656826569e-07,
+      "loss": 1.361,
+      "step": 2175
+    },
+    {
+      "epoch": 2.7081518357187306,
+      "grad_norm": 3.307027816772461,
+      "learning_rate": 5.742619926199262e-07,
+      "loss": 1.3509,
+      "step": 2176
+    },
+    {
+      "epoch": 2.7093963907902925,
+      "grad_norm": 3.4393889904022217,
+      "learning_rate": 5.719557195571956e-07,
+      "loss": 1.331,
+      "step": 2177
+    },
+    {
+      "epoch": 2.7106409458618543,
+      "grad_norm": 3.312872886657715,
+      "learning_rate": 5.696494464944649e-07,
+      "loss": 1.3541,
+      "step": 2178
+    },
+    {
+      "epoch": 2.711885500933416,
+      "grad_norm": 3.4782755374908447,
+      "learning_rate": 5.673431734317344e-07,
+      "loss": 1.3078,
+      "step": 2179
+    },
+    {
+      "epoch": 2.713130056004978,
+      "grad_norm": 3.457902193069458,
+      "learning_rate": 5.650369003690037e-07,
+      "loss": 1.3712,
+      "step": 2180
+    },
+    {
+      "epoch": 2.713130056004978,
+      "eval_loss": 1.6061460971832275,
+      "eval_runtime": 50.4797,
+      "eval_samples_per_second": 19.81,
+      "eval_steps_per_second": 0.832,
+      "step": 2180
+    },
+    {
+      "epoch": 2.71437461107654,
+      "grad_norm": 3.508249282836914,
+      "learning_rate": 5.627306273062731e-07,
+      "loss": 1.3948,
+      "step": 2181
+    },
+    {
+      "epoch": 2.715619166148102,
+      "grad_norm": 3.3043899536132812,
+      "learning_rate": 5.604243542435424e-07,
+      "loss": 1.3382,
+      "step": 2182
+    },
+    {
+      "epoch": 2.716863721219664,
+      "grad_norm": 3.4448275566101074,
+      "learning_rate": 5.581180811808118e-07,
+      "loss": 1.3422,
+      "step": 2183
+    },
+    {
+      "epoch": 2.718108276291226,
+      "grad_norm": 3.3538706302642822,
+      "learning_rate": 5.558118081180812e-07,
+      "loss": 1.3054,
+      "step": 2184
+    },
+    {
+      "epoch": 2.7193528313627877,
+      "grad_norm": 3.4720618724823,
+      "learning_rate": 5.535055350553506e-07,
+      "loss": 1.3525,
+      "step": 2185
+    },
+    {
+      "epoch": 2.7205973864343496,
+      "grad_norm": 3.370326519012451,
+      "learning_rate": 5.5119926199262e-07,
+      "loss": 1.3756,
+      "step": 2186
+    },
+    {
+      "epoch": 2.721841941505912,
+      "grad_norm": 3.4985008239746094,
+      "learning_rate": 5.488929889298893e-07,
+      "loss": 1.3509,
+      "step": 2187
+    },
+    {
+      "epoch": 2.7230864965774737,
+      "grad_norm": 3.3014514446258545,
+      "learning_rate": 5.465867158671587e-07,
+      "loss": 1.2928,
+      "step": 2188
+    },
+    {
+      "epoch": 2.7243310516490356,
+      "grad_norm": 3.491454839706421,
+      "learning_rate": 5.44280442804428e-07,
+      "loss": 1.315,
+      "step": 2189
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "grad_norm": 3.3876538276672363,
+      "learning_rate": 5.419741697416975e-07,
+      "loss": 1.3344,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "eval_loss": 1.606774926185608,
+      "eval_runtime": 49.375,
+      "eval_samples_per_second": 20.253,
+      "eval_steps_per_second": 0.851,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7268201617921592,
+      "grad_norm": 3.2686572074890137,
+      "learning_rate": 5.396678966789668e-07,
+      "loss": 1.3334,
+      "step": 2191
+    },
+    {
+      "epoch": 2.728064716863721,
+      "grad_norm": 3.539928913116455,
+      "learning_rate": 5.373616236162362e-07,
+      "loss": 1.3676,
+      "step": 2192
+    },
+    {
+      "epoch": 2.729309271935283,
+      "grad_norm": 3.424800395965576,
+      "learning_rate": 5.350553505535055e-07,
+      "loss": 1.3207,
+      "step": 2193
+    },
+    {
+      "epoch": 2.730553827006845,
+      "grad_norm": 3.459237575531006,
+      "learning_rate": 5.32749077490775e-07,
+      "loss": 1.3459,
+      "step": 2194
+    },
+    {
+      "epoch": 2.731798382078407,
+      "grad_norm": 3.39109468460083,
+      "learning_rate": 5.304428044280444e-07,
+      "loss": 1.3408,
+      "step": 2195
+    },
+    {
+      "epoch": 2.733042937149969,
+      "grad_norm": 3.4368112087249756,
+      "learning_rate": 5.281365313653137e-07,
+      "loss": 1.2679,
+      "step": 2196
+    },
+    {
+      "epoch": 2.734287492221531,
+      "grad_norm": 3.3579020500183105,
+      "learning_rate": 5.258302583025831e-07,
+      "loss": 1.3531,
+      "step": 2197
+    },
+    {
+      "epoch": 2.7355320472930926,
+      "grad_norm": 3.2895281314849854,
+      "learning_rate": 5.235239852398524e-07,
+      "loss": 1.3127,
+      "step": 2198
+    },
+    {
+      "epoch": 2.7367766023646545,
+      "grad_norm": 3.546879529953003,
+      "learning_rate": 5.212177121771219e-07,
+      "loss": 1.3816,
+      "step": 2199
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "grad_norm": 3.4015355110168457,
+      "learning_rate": 5.189114391143912e-07,
+      "loss": 1.3026,
+      "step": 2200
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "eval_loss": 1.6100414991378784,
+      "eval_runtime": 44.238,
+      "eval_samples_per_second": 22.605,
+      "eval_steps_per_second": 0.949,
+      "step": 2200
+    },
+    {
+      "epoch": 2.7392657125077786,
+      "grad_norm": 3.4278082847595215,
+      "learning_rate": 5.166051660516606e-07,
+      "loss": 1.3545,
+      "step": 2201
+    },
+    {
+      "epoch": 2.7405102675793405,
+      "grad_norm": 3.4757940769195557,
+      "learning_rate": 5.142988929889299e-07,
+      "loss": 1.3632,
+      "step": 2202
+    },
+    {
+      "epoch": 2.7417548226509023,
+      "grad_norm": 3.419520854949951,
+      "learning_rate": 5.119926199261993e-07,
+      "loss": 1.3591,
+      "step": 2203
+    },
+    {
+      "epoch": 2.742999377722464,
+      "grad_norm": 3.348172903060913,
+      "learning_rate": 5.096863468634687e-07,
+      "loss": 1.3218,
+      "step": 2204
+    },
+    {
+      "epoch": 2.744243932794026,
+      "grad_norm": 3.4806814193725586,
+      "learning_rate": 5.073800738007381e-07,
+      "loss": 1.3907,
+      "step": 2205
+    },
+    {
+      "epoch": 2.745488487865588,
+      "grad_norm": 3.465930700302124,
+      "learning_rate": 5.050738007380075e-07,
+      "loss": 1.3458,
+      "step": 2206
+    },
+    {
+      "epoch": 2.7467330429371497,
+      "grad_norm": 3.377190351486206,
+      "learning_rate": 5.027675276752768e-07,
+      "loss": 1.3265,
+      "step": 2207
+    },
+    {
+      "epoch": 2.747977598008712,
+      "grad_norm": 3.2912864685058594,
+      "learning_rate": 5.004612546125461e-07,
+      "loss": 1.3413,
+      "step": 2208
+    },
+    {
+      "epoch": 2.749222153080274,
+      "grad_norm": 3.4280965328216553,
+      "learning_rate": 4.981549815498156e-07,
+      "loss": 1.3274,
+      "step": 2209
+    },
+    {
+      "epoch": 2.7504667081518357,
+      "grad_norm": 3.232537031173706,
+      "learning_rate": 4.958487084870849e-07,
+      "loss": 1.2905,
+      "step": 2210
+    },
+    {
+      "epoch": 2.7504667081518357,
+      "eval_loss": 1.6082451343536377,
+      "eval_runtime": 41.3326,
+      "eval_samples_per_second": 24.194,
+      "eval_steps_per_second": 1.016,
+      "step": 2210
+    },
+    {
+      "epoch": 2.7517112632233975,
+      "grad_norm": 3.5405726432800293,
+      "learning_rate": 4.935424354243543e-07,
+      "loss": 1.3306,
+      "step": 2211
+    },
+    {
+      "epoch": 2.7529558182949594,
+      "grad_norm": 3.414891481399536,
+      "learning_rate": 4.912361623616236e-07,
+      "loss": 1.3425,
+      "step": 2212
+    },
+    {
+      "epoch": 2.7542003733665217,
+      "grad_norm": 3.4994254112243652,
+      "learning_rate": 4.88929889298893e-07,
+      "loss": 1.3429,
+      "step": 2213
+    },
+    {
+      "epoch": 2.7554449284380835,
+      "grad_norm": 3.3640542030334473,
+      "learning_rate": 4.866236162361624e-07,
+      "loss": 1.3247,
+      "step": 2214
+    },
+    {
+      "epoch": 2.7566894835096454,
+      "grad_norm": 3.4134693145751953,
+      "learning_rate": 4.843173431734318e-07,
+      "loss": 1.3437,
+      "step": 2215
+    },
+    {
+      "epoch": 2.7579340385812072,
+      "grad_norm": 3.4103310108184814,
+      "learning_rate": 4.820110701107011e-07,
+      "loss": 1.3612,
+      "step": 2216
+    },
+    {
+      "epoch": 2.759178593652769,
+      "grad_norm": 3.3797495365142822,
+      "learning_rate": 4.797047970479705e-07,
+      "loss": 1.3379,
+      "step": 2217
+    },
+    {
+      "epoch": 2.760423148724331,
+      "grad_norm": 3.40238094329834,
+      "learning_rate": 4.773985239852398e-07,
+      "loss": 1.3301,
+      "step": 2218
+    },
+    {
+      "epoch": 2.7616677037958928,
+      "grad_norm": 3.350519895553589,
+      "learning_rate": 4.7509225092250926e-07,
+      "loss": 1.3228,
+      "step": 2219
+    },
+    {
+      "epoch": 2.7629122588674546,
+      "grad_norm": 3.2626607418060303,
+      "learning_rate": 4.7278597785977863e-07,
+      "loss": 1.3008,
+      "step": 2220
+    },
+    {
+      "epoch": 2.7629122588674546,
+      "eval_loss": 1.6074291467666626,
+      "eval_runtime": 43.4661,
+      "eval_samples_per_second": 23.006,
+      "eval_steps_per_second": 0.966,
+      "step": 2220
+    },
+    {
+      "epoch": 2.764156813939017,
+      "grad_norm": 3.508967161178589,
+      "learning_rate": 4.70479704797048e-07,
+      "loss": 1.3492,
+      "step": 2221
+    },
+    {
+      "epoch": 2.7654013690105788,
+      "grad_norm": 3.5330708026885986,
+      "learning_rate": 4.6817343173431737e-07,
+      "loss": 1.395,
+      "step": 2222
+    },
+    {
+      "epoch": 2.7666459240821406,
+      "grad_norm": 3.3957931995391846,
+      "learning_rate": 4.6586715867158674e-07,
+      "loss": 1.3331,
+      "step": 2223
+    },
+    {
+      "epoch": 2.7678904791537025,
+      "grad_norm": 3.3957931995391846,
+      "learning_rate": 4.6586715867158674e-07,
+      "loss": 1.3279,
+      "step": 2224
+    },
+    {
+      "epoch": 2.7691350342252643,
+      "grad_norm": 3.4120025634765625,
+      "learning_rate": 4.635608856088561e-07,
+      "loss": 1.3445,
+      "step": 2225
+    },
+    {
+      "epoch": 2.7703795892968266,
+      "grad_norm": 3.4226207733154297,
+      "learning_rate": 4.612546125461255e-07,
+      "loss": 1.2962,
+      "step": 2226
+    },
+    {
+      "epoch": 2.7716241443683884,
+      "grad_norm": 3.4244253635406494,
+      "learning_rate": 4.589483394833949e-07,
+      "loss": 1.3615,
+      "step": 2227
+    },
+    {
+      "epoch": 2.7728686994399503,
+      "grad_norm": 3.4812207221984863,
+      "learning_rate": 4.566420664206642e-07,
+      "loss": 1.3881,
+      "step": 2228
+    },
+    {
+      "epoch": 2.774113254511512,
+      "grad_norm": 3.3681764602661133,
+      "learning_rate": 4.5433579335793364e-07,
+      "loss": 1.3652,
+      "step": 2229
+    },
+    {
+      "epoch": 2.775357809583074,
+      "grad_norm": 3.3974297046661377,
+      "learning_rate": 4.5202952029520296e-07,
+      "loss": 1.303,
+      "step": 2230
+    },
+    {
+      "epoch": 2.775357809583074,
+      "eval_loss": 1.6103951930999756,
+      "eval_runtime": 44.0184,
+      "eval_samples_per_second": 22.718,
+      "eval_steps_per_second": 0.954,
+      "step": 2230
+    },
+    {
+      "epoch": 2.776602364654636,
+      "grad_norm": 3.2922801971435547,
+      "learning_rate": 4.497232472324724e-07,
+      "loss": 1.3118,
+      "step": 2231
+    },
+    {
+      "epoch": 2.7778469197261977,
+      "grad_norm": 3.332679271697998,
+      "learning_rate": 4.474169741697417e-07,
+      "loss": 1.3506,
+      "step": 2232
+    },
+    {
+      "epoch": 2.7790914747977595,
+      "grad_norm": 3.288560628890991,
+      "learning_rate": 4.451107011070111e-07,
+      "loss": 1.3231,
+      "step": 2233
+    },
+    {
+      "epoch": 2.780336029869322,
+      "grad_norm": 3.3787007331848145,
+      "learning_rate": 4.4280442804428044e-07,
+      "loss": 1.3377,
+      "step": 2234
+    },
+    {
+      "epoch": 2.7815805849408837,
+      "grad_norm": 3.529020071029663,
+      "learning_rate": 4.4049815498154986e-07,
+      "loss": 1.3268,
+      "step": 2235
+    },
+    {
+      "epoch": 2.7828251400124455,
+      "grad_norm": 3.3797807693481445,
+      "learning_rate": 4.3819188191881923e-07,
+      "loss": 1.3024,
+      "step": 2236
+    },
+    {
+      "epoch": 2.7840696950840074,
+      "grad_norm": 3.497357130050659,
+      "learning_rate": 4.358856088560886e-07,
+      "loss": 1.3452,
+      "step": 2237
+    },
+    {
+      "epoch": 2.7853142501555697,
+      "grad_norm": 3.4761931896209717,
+      "learning_rate": 4.3357933579335797e-07,
+      "loss": 1.3683,
+      "step": 2238
+    },
+    {
+      "epoch": 2.7865588052271315,
+      "grad_norm": 3.2924118041992188,
+      "learning_rate": 4.3127306273062734e-07,
+      "loss": 1.3321,
+      "step": 2239
+    },
+    {
+      "epoch": 2.7878033602986934,
+      "grad_norm": 3.4957733154296875,
+      "learning_rate": 4.289667896678967e-07,
+      "loss": 1.3366,
+      "step": 2240
+    },
+    {
+      "epoch": 2.7878033602986934,
+      "eval_loss": 1.6076867580413818,
+      "eval_runtime": 43.7189,
+      "eval_samples_per_second": 22.873,
+      "eval_steps_per_second": 0.961,
+      "step": 2240
+    },
+    {
+      "epoch": 2.789047915370255,
+      "grad_norm": 3.566307544708252,
+      "learning_rate": 4.266605166051661e-07,
+      "loss": 1.3584,
+      "step": 2241
+    },
+    {
+      "epoch": 2.790292470441817,
+      "grad_norm": 3.6012961864471436,
+      "learning_rate": 4.2435424354243545e-07,
+      "loss": 1.3666,
+      "step": 2242
+    },
+    {
+      "epoch": 2.791537025513379,
+      "grad_norm": 3.5159049034118652,
+      "learning_rate": 4.220479704797048e-07,
+      "loss": 1.3277,
+      "step": 2243
+    },
+    {
+      "epoch": 2.7927815805849407,
+      "grad_norm": 3.366673469543457,
+      "learning_rate": 4.197416974169742e-07,
+      "loss": 1.3685,
+      "step": 2244
+    },
+    {
+      "epoch": 2.7940261356565026,
+      "grad_norm": 3.439293384552002,
+      "learning_rate": 4.1743542435424356e-07,
+      "loss": 1.3611,
+      "step": 2245
+    },
+    {
+      "epoch": 2.7952706907280644,
+      "grad_norm": 3.3428847789764404,
+      "learning_rate": 4.1512915129151293e-07,
+      "loss": 1.3344,
+      "step": 2246
+    },
+    {
+      "epoch": 2.7965152457996267,
+      "grad_norm": 3.3846447467803955,
+      "learning_rate": 4.1282287822878235e-07,
+      "loss": 1.3298,
+      "step": 2247
+    },
+    {
+      "epoch": 2.7977598008711886,
+      "grad_norm": 3.4208669662475586,
+      "learning_rate": 4.1051660516605167e-07,
+      "loss": 1.3762,
+      "step": 2248
+    },
+    {
+      "epoch": 2.7990043559427504,
+      "grad_norm": 3.3218090534210205,
+      "learning_rate": 4.082103321033211e-07,
+      "loss": 1.2945,
+      "step": 2249
+    },
+    {
+      "epoch": 2.8002489110143123,
+      "grad_norm": 3.4152491092681885,
+      "learning_rate": 4.059040590405904e-07,
+      "loss": 1.3234,
+      "step": 2250
+    },
+    {
+      "epoch": 2.8002489110143123,
+      "eval_loss": 1.6081277132034302,
+      "eval_runtime": 43.957,
+      "eval_samples_per_second": 22.75,
+      "eval_steps_per_second": 0.955,
+      "step": 2250
+    },
+    {
+      "epoch": 2.8014934660858746,
+      "grad_norm": 3.408590316772461,
+      "learning_rate": 4.0359778597785983e-07,
+      "loss": 1.3457,
+      "step": 2251
+    },
+    {
+      "epoch": 2.8027380211574364,
+      "grad_norm": 3.53322172164917,
+      "learning_rate": 4.0129151291512915e-07,
+      "loss": 1.3905,
+      "step": 2252
+    },
+    {
+      "epoch": 2.8039825762289983,
+      "grad_norm": 3.3232524394989014,
+      "learning_rate": 3.9898523985239857e-07,
+      "loss": 1.298,
+      "step": 2253
+    },
+    {
+      "epoch": 2.80522713130056,
+      "grad_norm": 3.383641242980957,
+      "learning_rate": 3.966789667896679e-07,
+      "loss": 1.3425,
+      "step": 2254
+    },
+    {
+      "epoch": 2.806471686372122,
+      "grad_norm": 3.586348295211792,
+      "learning_rate": 3.943726937269373e-07,
+      "loss": 1.4114,
+      "step": 2255
+    },
+    {
+      "epoch": 2.807716241443684,
+      "grad_norm": 3.3169400691986084,
+      "learning_rate": 3.920664206642067e-07,
+      "loss": 1.3231,
+      "step": 2256
+    },
+    {
+      "epoch": 2.8089607965152457,
+      "grad_norm": 3.4631175994873047,
+      "learning_rate": 3.8976014760147605e-07,
+      "loss": 1.3547,
+      "step": 2257
+    },
+    {
+      "epoch": 2.8102053515868075,
+      "grad_norm": 3.609144687652588,
+      "learning_rate": 3.874538745387454e-07,
+      "loss": 1.3917,
+      "step": 2258
+    },
+    {
+      "epoch": 2.8114499066583694,
+      "grad_norm": 3.5235679149627686,
+      "learning_rate": 3.851476014760148e-07,
+      "loss": 1.358,
+      "step": 2259
+    },
+    {
+      "epoch": 2.8126944617299316,
+      "grad_norm": 3.3768770694732666,
+      "learning_rate": 3.8284132841328416e-07,
+      "loss": 1.39,
+      "step": 2260
+    },
+    {
+      "epoch": 2.8126944617299316,
+      "eval_loss": 1.6079868078231812,
+      "eval_runtime": 45.6946,
+      "eval_samples_per_second": 21.884,
+      "eval_steps_per_second": 0.919,
+      "step": 2260
+    },
+    {
+      "epoch": 2.8139390168014935,
+      "grad_norm": 3.3989570140838623,
+      "learning_rate": 3.8053505535055353e-07,
+      "loss": 1.3377,
+      "step": 2261
+    },
+    {
+      "epoch": 2.8151835718730553,
+      "grad_norm": 3.508310317993164,
+      "learning_rate": 3.782287822878229e-07,
+      "loss": 1.3407,
+      "step": 2262
+    },
+    {
+      "epoch": 2.816428126944617,
+      "grad_norm": 3.4873592853546143,
+      "learning_rate": 3.7592250922509227e-07,
+      "loss": 1.3646,
+      "step": 2263
+    },
+    {
+      "epoch": 2.8176726820161795,
+      "grad_norm": 3.4191977977752686,
+      "learning_rate": 3.7361623616236164e-07,
+      "loss": 1.311,
+      "step": 2264
+    },
+    {
+      "epoch": 2.8189172370877413,
+      "grad_norm": 3.394169330596924,
+      "learning_rate": 3.71309963099631e-07,
+      "loss": 1.333,
+      "step": 2265
+    },
+    {
+      "epoch": 2.820161792159303,
+      "grad_norm": 3.4663238525390625,
+      "learning_rate": 3.690036900369004e-07,
+      "loss": 1.3387,
+      "step": 2266
+    },
+    {
+      "epoch": 2.821406347230865,
+      "grad_norm": 3.4985597133636475,
+      "learning_rate": 3.666974169741698e-07,
+      "loss": 1.337,
+      "step": 2267
+    },
+    {
+      "epoch": 2.822650902302427,
+      "grad_norm": 3.3662006855010986,
+      "learning_rate": 3.643911439114391e-07,
+      "loss": 1.3339,
+      "step": 2268
+    },
+    {
+      "epoch": 2.8238954573739887,
+      "grad_norm": 3.384871482849121,
+      "learning_rate": 3.6208487084870854e-07,
+      "loss": 1.3116,
+      "step": 2269
+    },
+    {
+      "epoch": 2.8251400124455506,
+      "grad_norm": 3.3345062732696533,
+      "learning_rate": 3.5977859778597786e-07,
+      "loss": 1.2856,
+      "step": 2270
+    },
+    {
+      "epoch": 2.8251400124455506,
+      "eval_loss": 1.6083563566207886,
+      "eval_runtime": 46.5813,
+      "eval_samples_per_second": 21.468,
+      "eval_steps_per_second": 0.902,
+      "step": 2270
+    },
+    {
+      "epoch": 2.8263845675171124,
+      "grad_norm": 3.531426429748535,
+      "learning_rate": 3.574723247232473e-07,
+      "loss": 1.3625,
+      "step": 2271
+    },
+    {
+      "epoch": 2.8276291225886747,
+      "grad_norm": 3.508230447769165,
+      "learning_rate": 3.551660516605166e-07,
+      "loss": 1.3397,
+      "step": 2272
+    },
+    {
+      "epoch": 2.8288736776602366,
+      "grad_norm": 3.392129421234131,
+      "learning_rate": 3.52859778597786e-07,
+      "loss": 1.3,
+      "step": 2273
+    },
+    {
+      "epoch": 2.8301182327317984,
+      "grad_norm": 3.501406192779541,
+      "learning_rate": 3.5055350553505534e-07,
+      "loss": 1.3431,
+      "step": 2274
+    },
+    {
+      "epoch": 2.8313627878033603,
+      "grad_norm": 3.3904361724853516,
+      "learning_rate": 3.4824723247232476e-07,
+      "loss": 1.3497,
+      "step": 2275
+    },
+    {
+      "epoch": 2.832607342874922,
+      "grad_norm": 3.5180835723876953,
+      "learning_rate": 3.4594095940959413e-07,
+      "loss": 1.3638,
+      "step": 2276
+    },
+    {
+      "epoch": 2.8338518979464844,
+      "grad_norm": 3.3576693534851074,
+      "learning_rate": 3.436346863468635e-07,
+      "loss": 1.3294,
+      "step": 2277
+    },
+    {
+      "epoch": 2.8350964530180462,
+      "grad_norm": 3.4021854400634766,
+      "learning_rate": 3.413284132841329e-07,
+      "loss": 1.3155,
+      "step": 2278
+    },
+    {
+      "epoch": 2.836341008089608,
+      "grad_norm": 3.4220190048217773,
+      "learning_rate": 3.3902214022140224e-07,
+      "loss": 1.3419,
+      "step": 2279
+    },
+    {
+      "epoch": 2.83758556316117,
+      "grad_norm": 3.4883368015289307,
+      "learning_rate": 3.367158671586716e-07,
+      "loss": 1.3287,
+      "step": 2280
+    },
+    {
+      "epoch": 2.83758556316117,
+      "eval_loss": 1.6085011959075928,
+      "eval_runtime": 46.0344,
+      "eval_samples_per_second": 21.723,
+      "eval_steps_per_second": 0.912,
+      "step": 2280
+    },
+    {
+      "epoch": 2.838830118232732,
+      "grad_norm": 3.521838903427124,
+      "learning_rate": 3.34409594095941e-07,
+      "loss": 1.3278,
+      "step": 2281
+    },
+    {
+      "epoch": 2.8400746733042936,
+      "grad_norm": 3.420236587524414,
+      "learning_rate": 3.3210332103321035e-07,
+      "loss": 1.3021,
+      "step": 2282
+    },
+    {
+      "epoch": 2.8413192283758555,
+      "grad_norm": 3.2946665287017822,
+      "learning_rate": 3.297970479704797e-07,
+      "loss": 1.3223,
+      "step": 2283
+    },
+    {
+      "epoch": 2.8425637834474173,
+      "grad_norm": 3.435534715652466,
+      "learning_rate": 3.274907749077491e-07,
+      "loss": 1.358,
+      "step": 2284
+    },
+    {
+      "epoch": 2.8438083385189796,
+      "grad_norm": 3.48772931098938,
+      "learning_rate": 3.2518450184501846e-07,
+      "loss": 1.3427,
+      "step": 2285
+    },
+    {
+      "epoch": 2.8450528935905415,
+      "grad_norm": 3.41283917427063,
+      "learning_rate": 3.2287822878228783e-07,
+      "loss": 1.3057,
+      "step": 2286
+    },
+    {
+      "epoch": 2.8462974486621033,
+      "grad_norm": 3.2920539379119873,
+      "learning_rate": 3.2057195571955726e-07,
+      "loss": 1.3073,
+      "step": 2287
+    },
+    {
+      "epoch": 2.847542003733665,
+      "grad_norm": 3.385733127593994,
+      "learning_rate": 3.182656826568266e-07,
+      "loss": 1.3433,
+      "step": 2288
+    },
+    {
+      "epoch": 2.848786558805227,
+      "grad_norm": 3.486422538757324,
+      "learning_rate": 3.15959409594096e-07,
+      "loss": 1.373,
+      "step": 2289
+    },
+    {
+      "epoch": 2.8500311138767893,
+      "grad_norm": 3.4137535095214844,
+      "learning_rate": 3.136531365313653e-07,
+      "loss": 1.3352,
+      "step": 2290
+    },
+    {
+      "epoch": 2.8500311138767893,
+      "eval_loss": 1.607330322265625,
+      "eval_runtime": 45.4501,
+      "eval_samples_per_second": 22.002,
+      "eval_steps_per_second": 0.924,
+      "step": 2290
+    },
+    {
+      "epoch": 2.851275668948351,
+      "grad_norm": 3.474705219268799,
+      "learning_rate": 3.1134686346863474e-07,
+      "loss": 1.3808,
+      "step": 2291
+    },
+    {
+      "epoch": 2.852520224019913,
+      "grad_norm": 3.299457311630249,
+      "learning_rate": 3.090405904059041e-07,
+      "loss": 1.3343,
+      "step": 2292
+    },
+    {
+      "epoch": 2.853764779091475,
+      "grad_norm": 3.265301465988159,
+      "learning_rate": 3.067343173431735e-07,
+      "loss": 1.3106,
+      "step": 2293
+    },
+    {
+      "epoch": 2.8550093341630367,
+      "grad_norm": 3.213136911392212,
+      "learning_rate": 3.0442804428044285e-07,
+      "loss": 1.3102,
+      "step": 2294
+    },
+    {
+      "epoch": 2.8562538892345986,
+      "grad_norm": 3.365297794342041,
+      "learning_rate": 3.021217712177122e-07,
+      "loss": 1.323,
+      "step": 2295
+    },
+    {
+      "epoch": 2.8574984443061604,
+      "grad_norm": 3.331441879272461,
+      "learning_rate": 2.998154981549816e-07,
+      "loss": 1.2881,
+      "step": 2296
+    },
+    {
+      "epoch": 2.8587429993777222,
+      "grad_norm": 3.4592764377593994,
+      "learning_rate": 2.9750922509225096e-07,
+      "loss": 1.3134,
+      "step": 2297
+    },
+    {
+      "epoch": 2.8599875544492845,
+      "grad_norm": 3.5126025676727295,
+      "learning_rate": 2.9520295202952033e-07,
+      "loss": 1.3614,
+      "step": 2298
+    },
+    {
+      "epoch": 2.8612321095208464,
+      "grad_norm": 3.344198226928711,
+      "learning_rate": 2.928966789667897e-07,
+      "loss": 1.3389,
+      "step": 2299
+    },
+    {
+      "epoch": 2.8624766645924082,
+      "grad_norm": 3.404078722000122,
+      "learning_rate": 2.9059040590405907e-07,
+      "loss": 1.3592,
+      "step": 2300
+    },
+    {
+      "epoch": 2.8624766645924082,
+      "eval_loss": 1.608008623123169,
+      "eval_runtime": 47.7633,
+      "eval_samples_per_second": 20.937,
+      "eval_steps_per_second": 0.879,
+      "step": 2300
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2409,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.169811641532416e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2300/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/latest
new file mode 100644
index 0000000000000000000000000000000000000000..361f8a1a19370f3a8801a82732f02b0e9e639e14
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/latest
@@ -0,0 +1 @@
+global_step2400
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7cebe108f7c295f538afca780f8bff98aa6232f2
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c7105b68cc7706a616e5184c049404c7551d0ad422b5b1a292aa19facaa3d0f
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..618e221236c3debbaad7ada8aebf8a3335d91f6e
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a994ddb08d50eccc51680080b7c8b587407a3900fb66b09b933d12402a0da4b
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..f378850538621142c5cd15d9d6f50bc9a0f23b02
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd0c5d9fbbbbb845dde3ee7d26db4ac79f6c6261f597d751fb6b57476387a4b
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7bc82fd4d98b0f19f25c22b5ddf7615b232a513c
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dccc9955c9da77cf93710a6e32fad3c8cc56a96d937c2c6502577f6b112a968a
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6
+size 17210734
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/tokenizer_config.json
@@ -0,0 +1,2086 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅂",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128258": {
+      "content": "🄿",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b8e8023bcd0f9cfdadc5bf8d8a3ac4b0dad4c84
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/trainer_state.json
@@ -0,0 +1,18753 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9869321717485997,
+  "eval_steps": 10,
+  "global_step": 2400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8368,
+      "step": 1
+    },
+    {
+      "epoch": 0.002489110143123833,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.86,
+      "step": 2
+    },
+    {
+      "epoch": 0.00373366521468575,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8265,
+      "step": 3
+    },
+    {
+      "epoch": 0.004978220286247666,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8271,
+      "step": 4
+    },
+    {
+      "epoch": 0.006222775357809583,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8768,
+      "step": 5
+    },
+    {
+      "epoch": 0.0074673304293715,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8402,
+      "step": 6
+    },
+    {
+      "epoch": 0.008711885500933417,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.785,
+      "step": 7
+    },
+    {
+      "epoch": 0.009956440572495333,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8466,
+      "step": 8
+    },
+    {
+      "epoch": 0.01120099564405725,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.7673,
+      "step": 9
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "eval_loss": 2.8535053730010986,
+      "eval_runtime": 42.446,
+      "eval_samples_per_second": 23.559,
+      "eval_steps_per_second": 0.989,
+      "step": 10
+    },
+    {
+      "epoch": 0.013690105787181083,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8592,
+      "step": 11
+    },
+    {
+      "epoch": 0.014934660858743,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.804,
+      "step": 12
+    },
+    {
+      "epoch": 0.016179215930304917,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8076,
+      "step": 13
+    },
+    {
+      "epoch": 0.017423771001866834,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8219,
+      "step": 14
+    },
+    {
+      "epoch": 0.018668326073428748,
+      "grad_norm": 5.105710506439209,
+      "learning_rate": 2.0746887966804982e-08,
+      "loss": 2.7995,
+      "step": 15
+    },
+    {
+      "epoch": 0.019912881144990666,
+      "grad_norm": 5.156588077545166,
+      "learning_rate": 4.1493775933609963e-08,
+      "loss": 2.8577,
+      "step": 16
+    },
+    {
+      "epoch": 0.021157436216552583,
+      "grad_norm": 5.709373950958252,
+      "learning_rate": 6.224066390041494e-08,
+      "loss": 2.8433,
+      "step": 17
+    },
+    {
+      "epoch": 0.0224019912881145,
+      "grad_norm": 5.64658784866333,
+      "learning_rate": 8.298755186721993e-08,
+      "loss": 2.8428,
+      "step": 18
+    },
+    {
+      "epoch": 0.023646546359676415,
+      "grad_norm": 5.541937351226807,
+      "learning_rate": 1.037344398340249e-07,
+      "loss": 2.7785,
+      "step": 19
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "grad_norm": 5.233374118804932,
+      "learning_rate": 1.2448132780082988e-07,
+      "loss": 2.8599,
+      "step": 20
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "eval_loss": 2.8521182537078857,
+      "eval_runtime": 43.5664,
+      "eval_samples_per_second": 22.953,
+      "eval_steps_per_second": 0.964,
+      "step": 20
+    },
+    {
+      "epoch": 0.02613565650280025,
+      "grad_norm": 5.407598972320557,
+      "learning_rate": 1.4522821576763488e-07,
+      "loss": 2.8081,
+      "step": 21
+    },
+    {
+      "epoch": 0.027380211574362167,
+      "grad_norm": 5.0483317375183105,
+      "learning_rate": 1.6597510373443985e-07,
+      "loss": 2.8489,
+      "step": 22
+    },
+    {
+      "epoch": 0.02862476664592408,
+      "grad_norm": 4.985588550567627,
+      "learning_rate": 1.8672199170124483e-07,
+      "loss": 2.8375,
+      "step": 23
+    },
+    {
+      "epoch": 0.029869321717486,
+      "grad_norm": 5.201700687408447,
+      "learning_rate": 2.074688796680498e-07,
+      "loss": 2.8355,
+      "step": 24
+    },
+    {
+      "epoch": 0.031113876789047916,
+      "grad_norm": 5.009235382080078,
+      "learning_rate": 2.2821576763485478e-07,
+      "loss": 2.7774,
+      "step": 25
+    },
+    {
+      "epoch": 0.03235843186060983,
+      "grad_norm": 4.509210109710693,
+      "learning_rate": 2.4896265560165975e-07,
+      "loss": 2.7992,
+      "step": 26
+    },
+    {
+      "epoch": 0.03360298693217175,
+      "grad_norm": 4.529262065887451,
+      "learning_rate": 2.6970954356846476e-07,
+      "loss": 2.836,
+      "step": 27
+    },
+    {
+      "epoch": 0.03484754200373367,
+      "grad_norm": 4.526787281036377,
+      "learning_rate": 2.9045643153526976e-07,
+      "loss": 2.8128,
+      "step": 28
+    },
+    {
+      "epoch": 0.03609209707529558,
+      "grad_norm": 4.016609191894531,
+      "learning_rate": 3.112033195020747e-07,
+      "loss": 2.7947,
+      "step": 29
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "grad_norm": 3.4784562587738037,
+      "learning_rate": 3.319502074688797e-07,
+      "loss": 2.7854,
+      "step": 30
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "eval_loss": 2.8020100593566895,
+      "eval_runtime": 44.8464,
+      "eval_samples_per_second": 22.298,
+      "eval_steps_per_second": 0.937,
+      "step": 30
+    },
+    {
+      "epoch": 0.038581207218419414,
+      "grad_norm": 3.4887945652008057,
+      "learning_rate": 3.5269709543568466e-07,
+      "loss": 2.7908,
+      "step": 31
+    },
+    {
+      "epoch": 0.03982576228998133,
+      "grad_norm": 3.4533374309539795,
+      "learning_rate": 3.7344398340248966e-07,
+      "loss": 2.7358,
+      "step": 32
+    },
+    {
+      "epoch": 0.04107031736154325,
+      "grad_norm": 3.270954132080078,
+      "learning_rate": 3.941908713692946e-07,
+      "loss": 2.8013,
+      "step": 33
+    },
+    {
+      "epoch": 0.042314872433105166,
+      "grad_norm": 3.61322283744812,
+      "learning_rate": 4.149377593360996e-07,
+      "loss": 2.7521,
+      "step": 34
+    },
+    {
+      "epoch": 0.043559427504667084,
+      "grad_norm": 3.158719062805176,
+      "learning_rate": 4.3568464730290456e-07,
+      "loss": 2.7573,
+      "step": 35
+    },
+    {
+      "epoch": 0.044803982576229,
+      "grad_norm": 2.888746500015259,
+      "learning_rate": 4.5643153526970956e-07,
+      "loss": 2.7795,
+      "step": 36
+    },
+    {
+      "epoch": 0.04604853764779091,
+      "grad_norm": 3.022629499435425,
+      "learning_rate": 4.771784232365145e-07,
+      "loss": 2.7891,
+      "step": 37
+    },
+    {
+      "epoch": 0.04729309271935283,
+      "grad_norm": 3.038097381591797,
+      "learning_rate": 4.979253112033195e-07,
+      "loss": 2.7376,
+      "step": 38
+    },
+    {
+      "epoch": 0.04853764779091475,
+      "grad_norm": 2.9992239475250244,
+      "learning_rate": 5.186721991701245e-07,
+      "loss": 2.7279,
+      "step": 39
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "grad_norm": 2.908842086791992,
+      "learning_rate": 5.394190871369295e-07,
+      "loss": 2.6582,
+      "step": 40
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "eval_loss": 2.725968837738037,
+      "eval_runtime": 44.9498,
+      "eval_samples_per_second": 22.247,
+      "eval_steps_per_second": 0.934,
+      "step": 40
+    },
+    {
+      "epoch": 0.05102675793403858,
+      "grad_norm": 2.8025174140930176,
+      "learning_rate": 5.601659751037345e-07,
+      "loss": 2.6809,
+      "step": 41
+    },
+    {
+      "epoch": 0.0522713130056005,
+      "grad_norm": 2.7555699348449707,
+      "learning_rate": 5.809128630705395e-07,
+      "loss": 2.6954,
+      "step": 42
+    },
+    {
+      "epoch": 0.053515868077162417,
+      "grad_norm": 2.4732933044433594,
+      "learning_rate": 6.016597510373444e-07,
+      "loss": 2.7307,
+      "step": 43
+    },
+    {
+      "epoch": 0.054760423148724334,
+      "grad_norm": 2.451366424560547,
+      "learning_rate": 6.224066390041494e-07,
+      "loss": 2.7064,
+      "step": 44
+    },
+    {
+      "epoch": 0.056004978220286245,
+      "grad_norm": 2.5768678188323975,
+      "learning_rate": 6.431535269709543e-07,
+      "loss": 2.6466,
+      "step": 45
+    },
+    {
+      "epoch": 0.05724953329184816,
+      "grad_norm": 2.579332113265991,
+      "learning_rate": 6.639004149377594e-07,
+      "loss": 2.6127,
+      "step": 46
+    },
+    {
+      "epoch": 0.05849408836341008,
+      "grad_norm": 2.231207847595215,
+      "learning_rate": 6.846473029045644e-07,
+      "loss": 2.6429,
+      "step": 47
+    },
+    {
+      "epoch": 0.059738643434972,
+      "grad_norm": 2.134375810623169,
+      "learning_rate": 7.053941908713693e-07,
+      "loss": 2.6423,
+      "step": 48
+    },
+    {
+      "epoch": 0.060983198506533914,
+      "grad_norm": 2.0485494136810303,
+      "learning_rate": 7.261410788381744e-07,
+      "loss": 2.581,
+      "step": 49
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "grad_norm": 2.02909517288208,
+      "learning_rate": 7.468879668049793e-07,
+      "loss": 2.6357,
+      "step": 50
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "eval_loss": 2.632822036743164,
+      "eval_runtime": 47.0104,
+      "eval_samples_per_second": 21.272,
+      "eval_steps_per_second": 0.893,
+      "step": 50
+    },
+    {
+      "epoch": 0.06347230864965775,
+      "grad_norm": 1.892500877380371,
+      "learning_rate": 7.676348547717843e-07,
+      "loss": 2.5945,
+      "step": 51
+    },
+    {
+      "epoch": 0.06471686372121967,
+      "grad_norm": 2.107921600341797,
+      "learning_rate": 7.883817427385892e-07,
+      "loss": 2.5867,
+      "step": 52
+    },
+    {
+      "epoch": 0.06596141879278158,
+      "grad_norm": 1.8287527561187744,
+      "learning_rate": 8.091286307053943e-07,
+      "loss": 2.5862,
+      "step": 53
+    },
+    {
+      "epoch": 0.0672059738643435,
+      "grad_norm": 1.7342065572738647,
+      "learning_rate": 8.298755186721992e-07,
+      "loss": 2.5735,
+      "step": 54
+    },
+    {
+      "epoch": 0.06845052893590542,
+      "grad_norm": 1.7905038595199585,
+      "learning_rate": 8.506224066390042e-07,
+      "loss": 2.572,
+      "step": 55
+    },
+    {
+      "epoch": 0.06969508400746734,
+      "grad_norm": 1.9715144634246826,
+      "learning_rate": 8.713692946058091e-07,
+      "loss": 2.5797,
+      "step": 56
+    },
+    {
+      "epoch": 0.07093963907902924,
+      "grad_norm": 1.788966417312622,
+      "learning_rate": 8.921161825726142e-07,
+      "loss": 2.5567,
+      "step": 57
+    },
+    {
+      "epoch": 0.07218419415059116,
+      "grad_norm": 1.8978915214538574,
+      "learning_rate": 9.128630705394191e-07,
+      "loss": 2.5776,
+      "step": 58
+    },
+    {
+      "epoch": 0.07342874922215308,
+      "grad_norm": 1.6401689052581787,
+      "learning_rate": 9.336099585062241e-07,
+      "loss": 2.5662,
+      "step": 59
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "grad_norm": 1.8200798034667969,
+      "learning_rate": 9.54356846473029e-07,
+      "loss": 2.5001,
+      "step": 60
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "eval_loss": 2.569110870361328,
+      "eval_runtime": 47.3292,
+      "eval_samples_per_second": 21.129,
+      "eval_steps_per_second": 0.887,
+      "step": 60
+    },
+    {
+      "epoch": 0.07591785936527691,
+      "grad_norm": 1.5468674898147583,
+      "learning_rate": 9.751037344398341e-07,
+      "loss": 2.5308,
+      "step": 61
+    },
+    {
+      "epoch": 0.07716241443683883,
+      "grad_norm": 1.723684310913086,
+      "learning_rate": 9.95850622406639e-07,
+      "loss": 2.4907,
+      "step": 62
+    },
+    {
+      "epoch": 0.07840696950840075,
+      "grad_norm": 1.6346293687820435,
+      "learning_rate": 1.0165975103734441e-06,
+      "loss": 2.514,
+      "step": 63
+    },
+    {
+      "epoch": 0.07965152457996266,
+      "grad_norm": 1.8902088403701782,
+      "learning_rate": 1.037344398340249e-06,
+      "loss": 2.4859,
+      "step": 64
+    },
+    {
+      "epoch": 0.08089607965152458,
+      "grad_norm": 2.092611312866211,
+      "learning_rate": 1.058091286307054e-06,
+      "loss": 2.4888,
+      "step": 65
+    },
+    {
+      "epoch": 0.0821406347230865,
+      "grad_norm": 1.758482575416565,
+      "learning_rate": 1.078838174273859e-06,
+      "loss": 2.4537,
+      "step": 66
+    },
+    {
+      "epoch": 0.08338518979464841,
+      "grad_norm": 1.8941714763641357,
+      "learning_rate": 1.099585062240664e-06,
+      "loss": 2.5416,
+      "step": 67
+    },
+    {
+      "epoch": 0.08462974486621033,
+      "grad_norm": 1.7030234336853027,
+      "learning_rate": 1.120331950207469e-06,
+      "loss": 2.4791,
+      "step": 68
+    },
+    {
+      "epoch": 0.08587429993777225,
+      "grad_norm": 1.7602269649505615,
+      "learning_rate": 1.141078838174274e-06,
+      "loss": 2.4316,
+      "step": 69
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "grad_norm": 1.7635431289672852,
+      "learning_rate": 1.161825726141079e-06,
+      "loss": 2.4674,
+      "step": 70
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "eval_loss": 2.4922080039978027,
+      "eval_runtime": 45.3028,
+      "eval_samples_per_second": 22.074,
+      "eval_steps_per_second": 0.927,
+      "step": 70
+    },
+    {
+      "epoch": 0.08836341008089608,
+      "grad_norm": 2.0913803577423096,
+      "learning_rate": 1.182572614107884e-06,
+      "loss": 2.4341,
+      "step": 71
+    },
+    {
+      "epoch": 0.089607965152458,
+      "grad_norm": 1.8292183876037598,
+      "learning_rate": 1.2033195020746888e-06,
+      "loss": 2.4291,
+      "step": 72
+    },
+    {
+      "epoch": 0.09085252022401992,
+      "grad_norm": 2.206770181655884,
+      "learning_rate": 1.224066390041494e-06,
+      "loss": 2.3764,
+      "step": 73
+    },
+    {
+      "epoch": 0.09209707529558182,
+      "grad_norm": 1.8263559341430664,
+      "learning_rate": 1.2448132780082988e-06,
+      "loss": 2.4082,
+      "step": 74
+    },
+    {
+      "epoch": 0.09334163036714374,
+      "grad_norm": 1.9405455589294434,
+      "learning_rate": 1.2655601659751037e-06,
+      "loss": 2.4673,
+      "step": 75
+    },
+    {
+      "epoch": 0.09458618543870566,
+      "grad_norm": 2.138108015060425,
+      "learning_rate": 1.2863070539419086e-06,
+      "loss": 2.3879,
+      "step": 76
+    },
+    {
+      "epoch": 0.09583074051026758,
+      "grad_norm": 1.9024745225906372,
+      "learning_rate": 1.307053941908714e-06,
+      "loss": 2.4344,
+      "step": 77
+    },
+    {
+      "epoch": 0.0970752955818295,
+      "grad_norm": 1.9136689901351929,
+      "learning_rate": 1.3278008298755188e-06,
+      "loss": 2.474,
+      "step": 78
+    },
+    {
+      "epoch": 0.09831985065339141,
+      "grad_norm": 2.4752795696258545,
+      "learning_rate": 1.3485477178423237e-06,
+      "loss": 2.3412,
+      "step": 79
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "grad_norm": 1.8720006942749023,
+      "learning_rate": 1.3692946058091288e-06,
+      "loss": 2.3438,
+      "step": 80
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "eval_loss": 2.3980512619018555,
+      "eval_runtime": 48.2788,
+      "eval_samples_per_second": 20.713,
+      "eval_steps_per_second": 0.87,
+      "step": 80
+    },
+    {
+      "epoch": 0.10080896079651525,
+      "grad_norm": 2.671691656112671,
+      "learning_rate": 1.3900414937759337e-06,
+      "loss": 2.3336,
+      "step": 81
+    },
+    {
+      "epoch": 0.10205351586807716,
+      "grad_norm": 2.2953391075134277,
+      "learning_rate": 1.4107883817427386e-06,
+      "loss": 2.377,
+      "step": 82
+    },
+    {
+      "epoch": 0.10329807093963908,
+      "grad_norm": 3.009018898010254,
+      "learning_rate": 1.4315352697095435e-06,
+      "loss": 2.2977,
+      "step": 83
+    },
+    {
+      "epoch": 0.104542626011201,
+      "grad_norm": 2.664454936981201,
+      "learning_rate": 1.4522821576763488e-06,
+      "loss": 2.3271,
+      "step": 84
+    },
+    {
+      "epoch": 0.10578718108276292,
+      "grad_norm": 3.017303705215454,
+      "learning_rate": 1.4730290456431537e-06,
+      "loss": 2.3251,
+      "step": 85
+    },
+    {
+      "epoch": 0.10703173615432483,
+      "grad_norm": 2.634716510772705,
+      "learning_rate": 1.4937759336099586e-06,
+      "loss": 2.332,
+      "step": 86
+    },
+    {
+      "epoch": 0.10827629122588675,
+      "grad_norm": 3.059644937515259,
+      "learning_rate": 1.5145228215767635e-06,
+      "loss": 2.3478,
+      "step": 87
+    },
+    {
+      "epoch": 0.10952084629744867,
+      "grad_norm": 2.6962637901306152,
+      "learning_rate": 1.5352697095435686e-06,
+      "loss": 2.2792,
+      "step": 88
+    },
+    {
+      "epoch": 0.11076540136901059,
+      "grad_norm": 3.419729709625244,
+      "learning_rate": 1.5560165975103735e-06,
+      "loss": 2.2571,
+      "step": 89
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "grad_norm": 2.740781545639038,
+      "learning_rate": 1.5767634854771784e-06,
+      "loss": 2.2875,
+      "step": 90
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "eval_loss": 2.30843186378479,
+      "eval_runtime": 49.9264,
+      "eval_samples_per_second": 20.029,
+      "eval_steps_per_second": 0.841,
+      "step": 90
+    },
+    {
+      "epoch": 0.1132545115121344,
+      "grad_norm": 2.5608789920806885,
+      "learning_rate": 1.5975103734439833e-06,
+      "loss": 2.2592,
+      "step": 91
+    },
+    {
+      "epoch": 0.11449906658369632,
+      "grad_norm": 2.687999963760376,
+      "learning_rate": 1.6182572614107886e-06,
+      "loss": 2.2546,
+      "step": 92
+    },
+    {
+      "epoch": 0.11574362165525824,
+      "grad_norm": 2.695909023284912,
+      "learning_rate": 1.6390041493775935e-06,
+      "loss": 2.2525,
+      "step": 93
+    },
+    {
+      "epoch": 0.11698817672682016,
+      "grad_norm": 2.818357467651367,
+      "learning_rate": 1.6597510373443984e-06,
+      "loss": 2.216,
+      "step": 94
+    },
+    {
+      "epoch": 0.11823273179838208,
+      "grad_norm": 2.884119987487793,
+      "learning_rate": 1.6804979253112035e-06,
+      "loss": 2.2321,
+      "step": 95
+    },
+    {
+      "epoch": 0.119477286869944,
+      "grad_norm": 2.52104115486145,
+      "learning_rate": 1.7012448132780084e-06,
+      "loss": 2.199,
+      "step": 96
+    },
+    {
+      "epoch": 0.12072184194150591,
+      "grad_norm": 2.420313596725464,
+      "learning_rate": 1.7219917012448133e-06,
+      "loss": 2.1862,
+      "step": 97
+    },
+    {
+      "epoch": 0.12196639701306783,
+      "grad_norm": 2.8047542572021484,
+      "learning_rate": 1.7427385892116182e-06,
+      "loss": 2.1793,
+      "step": 98
+    },
+    {
+      "epoch": 0.12321095208462975,
+      "grad_norm": 2.836482286453247,
+      "learning_rate": 1.7634854771784235e-06,
+      "loss": 2.2271,
+      "step": 99
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "grad_norm": 2.5282301902770996,
+      "learning_rate": 1.7842323651452284e-06,
+      "loss": 2.1768,
+      "step": 100
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "eval_loss": 2.2312686443328857,
+      "eval_runtime": 49.2382,
+      "eval_samples_per_second": 20.309,
+      "eval_steps_per_second": 0.853,
+      "step": 100
+    },
+    {
+      "epoch": 0.12570006222775357,
+      "grad_norm": 3.3407280445098877,
+      "learning_rate": 1.8049792531120333e-06,
+      "loss": 2.1666,
+      "step": 101
+    },
+    {
+      "epoch": 0.1269446172993155,
+      "grad_norm": 2.4754133224487305,
+      "learning_rate": 1.8257261410788382e-06,
+      "loss": 2.1768,
+      "step": 102
+    },
+    {
+      "epoch": 0.1281891723708774,
+      "grad_norm": 3.430889129638672,
+      "learning_rate": 1.8464730290456433e-06,
+      "loss": 2.1953,
+      "step": 103
+    },
+    {
+      "epoch": 0.12943372744243933,
+      "grad_norm": 2.835294246673584,
+      "learning_rate": 1.8672199170124482e-06,
+      "loss": 2.146,
+      "step": 104
+    },
+    {
+      "epoch": 0.13067828251400124,
+      "grad_norm": 3.1532323360443115,
+      "learning_rate": 1.8879668049792531e-06,
+      "loss": 2.1729,
+      "step": 105
+    },
+    {
+      "epoch": 0.13192283758556317,
+      "grad_norm": 3.2278342247009277,
+      "learning_rate": 1.908713692946058e-06,
+      "loss": 2.1336,
+      "step": 106
+    },
+    {
+      "epoch": 0.13316739265712507,
+      "grad_norm": 2.7892515659332275,
+      "learning_rate": 1.929460580912863e-06,
+      "loss": 2.141,
+      "step": 107
+    },
+    {
+      "epoch": 0.134411947728687,
+      "grad_norm": 3.0179977416992188,
+      "learning_rate": 1.9502074688796682e-06,
+      "loss": 2.1255,
+      "step": 108
+    },
+    {
+      "epoch": 0.1356565028002489,
+      "grad_norm": 2.977935552597046,
+      "learning_rate": 1.970954356846473e-06,
+      "loss": 2.0986,
+      "step": 109
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "grad_norm": 3.3042492866516113,
+      "learning_rate": 1.991701244813278e-06,
+      "loss": 2.0992,
+      "step": 110
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "eval_loss": 2.171299695968628,
+      "eval_runtime": 42.9511,
+      "eval_samples_per_second": 23.282,
+      "eval_steps_per_second": 0.978,
+      "step": 110
+    },
+    {
+      "epoch": 0.13814561294337274,
+      "grad_norm": 2.9750890731811523,
+      "learning_rate": 2.012448132780083e-06,
+      "loss": 2.1116,
+      "step": 111
+    },
+    {
+      "epoch": 0.13939016801493467,
+      "grad_norm": 2.8199286460876465,
+      "learning_rate": 2.0331950207468883e-06,
+      "loss": 2.0822,
+      "step": 112
+    },
+    {
+      "epoch": 0.14063472308649658,
+      "grad_norm": 2.930532217025757,
+      "learning_rate": 2.053941908713693e-06,
+      "loss": 2.161,
+      "step": 113
+    },
+    {
+      "epoch": 0.14187927815805848,
+      "grad_norm": 3.3400321006774902,
+      "learning_rate": 2.074688796680498e-06,
+      "loss": 2.0953,
+      "step": 114
+    },
+    {
+      "epoch": 0.1431238332296204,
+      "grad_norm": 2.716564655303955,
+      "learning_rate": 2.095435684647303e-06,
+      "loss": 2.0757,
+      "step": 115
+    },
+    {
+      "epoch": 0.14436838830118232,
+      "grad_norm": 2.7586236000061035,
+      "learning_rate": 2.116182572614108e-06,
+      "loss": 2.0979,
+      "step": 116
+    },
+    {
+      "epoch": 0.14561294337274425,
+      "grad_norm": 2.622126340866089,
+      "learning_rate": 2.136929460580913e-06,
+      "loss": 2.0748,
+      "step": 117
+    },
+    {
+      "epoch": 0.14685749844430615,
+      "grad_norm": 2.646477460861206,
+      "learning_rate": 2.157676348547718e-06,
+      "loss": 2.0605,
+      "step": 118
+    },
+    {
+      "epoch": 0.14810205351586808,
+      "grad_norm": 2.8882410526275635,
+      "learning_rate": 2.178423236514523e-06,
+      "loss": 2.0501,
+      "step": 119
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "grad_norm": 2.8302247524261475,
+      "learning_rate": 2.199170124481328e-06,
+      "loss": 2.0592,
+      "step": 120
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "eval_loss": 2.1291966438293457,
+      "eval_runtime": 47.5399,
+      "eval_samples_per_second": 21.035,
+      "eval_steps_per_second": 0.883,
+      "step": 120
+    },
+    {
+      "epoch": 0.15059116365899192,
+      "grad_norm": 3.26979660987854,
+      "learning_rate": 2.219917012448133e-06,
+      "loss": 2.0422,
+      "step": 121
+    },
+    {
+      "epoch": 0.15183571873055382,
+      "grad_norm": 2.7483913898468018,
+      "learning_rate": 2.240663900414938e-06,
+      "loss": 2.0872,
+      "step": 122
+    },
+    {
+      "epoch": 0.15308027380211575,
+      "grad_norm": 3.1252024173736572,
+      "learning_rate": 2.2614107883817427e-06,
+      "loss": 2.0684,
+      "step": 123
+    },
+    {
+      "epoch": 0.15432482887367766,
+      "grad_norm": 2.9175283908843994,
+      "learning_rate": 2.282157676348548e-06,
+      "loss": 2.0522,
+      "step": 124
+    },
+    {
+      "epoch": 0.1555693839452396,
+      "grad_norm": 2.7950754165649414,
+      "learning_rate": 2.302904564315353e-06,
+      "loss": 2.065,
+      "step": 125
+    },
+    {
+      "epoch": 0.1568139390168015,
+      "grad_norm": 2.84049654006958,
+      "learning_rate": 2.323651452282158e-06,
+      "loss": 2.0781,
+      "step": 126
+    },
+    {
+      "epoch": 0.15805849408836342,
+      "grad_norm": 2.7396647930145264,
+      "learning_rate": 2.3443983402489627e-06,
+      "loss": 2.1758,
+      "step": 127
+    },
+    {
+      "epoch": 0.15930304915992533,
+      "grad_norm": 3.0765926837921143,
+      "learning_rate": 2.365145228215768e-06,
+      "loss": 2.0172,
+      "step": 128
+    },
+    {
+      "epoch": 0.16054760423148726,
+      "grad_norm": 3.217189073562622,
+      "learning_rate": 2.385892116182573e-06,
+      "loss": 2.0286,
+      "step": 129
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "grad_norm": 3.141545295715332,
+      "learning_rate": 2.4066390041493776e-06,
+      "loss": 2.0583,
+      "step": 130
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "eval_loss": 2.0946149826049805,
+      "eval_runtime": 54.92,
+      "eval_samples_per_second": 18.208,
+      "eval_steps_per_second": 0.765,
+      "step": 130
+    },
+    {
+      "epoch": 0.16303671437461106,
+      "grad_norm": 2.850052833557129,
+      "learning_rate": 2.4273858921161828e-06,
+      "loss": 2.0746,
+      "step": 131
+    },
+    {
+      "epoch": 0.164281269446173,
+      "grad_norm": 3.28913950920105,
+      "learning_rate": 2.448132780082988e-06,
+      "loss": 2.0539,
+      "step": 132
+    },
+    {
+      "epoch": 0.1655258245177349,
+      "grad_norm": 2.8819124698638916,
+      "learning_rate": 2.468879668049793e-06,
+      "loss": 1.9843,
+      "step": 133
+    },
+    {
+      "epoch": 0.16677037958929683,
+      "grad_norm": 3.6254632472991943,
+      "learning_rate": 2.4896265560165977e-06,
+      "loss": 2.0233,
+      "step": 134
+    },
+    {
+      "epoch": 0.16801493466085873,
+      "grad_norm": 2.7385146617889404,
+      "learning_rate": 2.5103734439834028e-06,
+      "loss": 1.9667,
+      "step": 135
+    },
+    {
+      "epoch": 0.16925948973242066,
+      "grad_norm": 2.9722647666931152,
+      "learning_rate": 2.5311203319502074e-06,
+      "loss": 1.9527,
+      "step": 136
+    },
+    {
+      "epoch": 0.17050404480398257,
+      "grad_norm": 3.139526605606079,
+      "learning_rate": 2.5518672199170125e-06,
+      "loss": 1.9967,
+      "step": 137
+    },
+    {
+      "epoch": 0.1717485998755445,
+      "grad_norm": 3.399920701980591,
+      "learning_rate": 2.5726141078838172e-06,
+      "loss": 1.9798,
+      "step": 138
+    },
+    {
+      "epoch": 0.1729931549471064,
+      "grad_norm": 2.8017327785491943,
+      "learning_rate": 2.5933609958506228e-06,
+      "loss": 1.983,
+      "step": 139
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "grad_norm": 3.9967198371887207,
+      "learning_rate": 2.614107883817428e-06,
+      "loss": 1.977,
+      "step": 140
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "eval_loss": 2.059037685394287,
+      "eval_runtime": 52.6232,
+      "eval_samples_per_second": 19.003,
+      "eval_steps_per_second": 0.798,
+      "step": 140
+    },
+    {
+      "epoch": 0.17548226509023024,
+      "grad_norm": 2.893092393875122,
+      "learning_rate": 2.6348547717842326e-06,
+      "loss": 1.9888,
+      "step": 141
+    },
+    {
+      "epoch": 0.17672682016179217,
+      "grad_norm": 3.634352207183838,
+      "learning_rate": 2.6556016597510377e-06,
+      "loss": 2.0726,
+      "step": 142
+    },
+    {
+      "epoch": 0.17797137523335407,
+      "grad_norm": 3.3651444911956787,
+      "learning_rate": 2.6763485477178423e-06,
+      "loss": 1.973,
+      "step": 143
+    },
+    {
+      "epoch": 0.179215930304916,
+      "grad_norm": 3.968986988067627,
+      "learning_rate": 2.6970954356846475e-06,
+      "loss": 1.9948,
+      "step": 144
+    },
+    {
+      "epoch": 0.1804604853764779,
+      "grad_norm": 3.20105242729187,
+      "learning_rate": 2.717842323651452e-06,
+      "loss": 1.9238,
+      "step": 145
+    },
+    {
+      "epoch": 0.18170504044803984,
+      "grad_norm": 3.648339033126831,
+      "learning_rate": 2.7385892116182577e-06,
+      "loss": 1.9923,
+      "step": 146
+    },
+    {
+      "epoch": 0.18294959551960174,
+      "grad_norm": 3.0856316089630127,
+      "learning_rate": 2.7593360995850628e-06,
+      "loss": 1.9749,
+      "step": 147
+    },
+    {
+      "epoch": 0.18419415059116365,
+      "grad_norm": 3.2681071758270264,
+      "learning_rate": 2.7800829875518675e-06,
+      "loss": 1.9304,
+      "step": 148
+    },
+    {
+      "epoch": 0.18543870566272558,
+      "grad_norm": 2.634958267211914,
+      "learning_rate": 2.8008298755186726e-06,
+      "loss": 1.9237,
+      "step": 149
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "grad_norm": 2.769491672515869,
+      "learning_rate": 2.8215767634854773e-06,
+      "loss": 1.8963,
+      "step": 150
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "eval_loss": 2.0394654273986816,
+      "eval_runtime": 55.2136,
+      "eval_samples_per_second": 18.111,
+      "eval_steps_per_second": 0.761,
+      "step": 150
+    },
+    {
+      "epoch": 0.1879278158058494,
+      "grad_norm": 3.4345781803131104,
+      "learning_rate": 2.8423236514522824e-06,
+      "loss": 1.9836,
+      "step": 151
+    },
+    {
+      "epoch": 0.18917237087741132,
+      "grad_norm": 3.115727424621582,
+      "learning_rate": 2.863070539419087e-06,
+      "loss": 1.9352,
+      "step": 152
+    },
+    {
+      "epoch": 0.19041692594897325,
+      "grad_norm": 3.050652503967285,
+      "learning_rate": 2.883817427385892e-06,
+      "loss": 1.9031,
+      "step": 153
+    },
+    {
+      "epoch": 0.19166148102053515,
+      "grad_norm": 2.9404428005218506,
+      "learning_rate": 2.9045643153526977e-06,
+      "loss": 1.9196,
+      "step": 154
+    },
+    {
+      "epoch": 0.19290603609209708,
+      "grad_norm": 3.080810785293579,
+      "learning_rate": 2.9253112033195024e-06,
+      "loss": 1.9405,
+      "step": 155
+    },
+    {
+      "epoch": 0.194150591163659,
+      "grad_norm": 3.058558702468872,
+      "learning_rate": 2.9460580912863075e-06,
+      "loss": 1.9052,
+      "step": 156
+    },
+    {
+      "epoch": 0.19539514623522092,
+      "grad_norm": 3.307955026626587,
+      "learning_rate": 2.966804979253112e-06,
+      "loss": 1.9245,
+      "step": 157
+    },
+    {
+      "epoch": 0.19663970130678282,
+      "grad_norm": 2.845506191253662,
+      "learning_rate": 2.9875518672199173e-06,
+      "loss": 1.9461,
+      "step": 158
+    },
+    {
+      "epoch": 0.19788425637834475,
+      "grad_norm": 2.758654832839966,
+      "learning_rate": 3.008298755186722e-06,
+      "loss": 1.8885,
+      "step": 159
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "grad_norm": 3.161252737045288,
+      "learning_rate": 3.029045643153527e-06,
+      "loss": 1.8535,
+      "step": 160
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "eval_loss": 2.018101453781128,
+      "eval_runtime": 51.0744,
+      "eval_samples_per_second": 19.579,
+      "eval_steps_per_second": 0.822,
+      "step": 160
+    },
+    {
+      "epoch": 0.2003733665214686,
+      "grad_norm": 3.196847677230835,
+      "learning_rate": 3.0497925311203326e-06,
+      "loss": 1.8853,
+      "step": 161
+    },
+    {
+      "epoch": 0.2016179215930305,
+      "grad_norm": 2.8364663124084473,
+      "learning_rate": 3.0705394190871373e-06,
+      "loss": 1.9463,
+      "step": 162
+    },
+    {
+      "epoch": 0.2028624766645924,
+      "grad_norm": 3.0874054431915283,
+      "learning_rate": 3.0912863070539424e-06,
+      "loss": 1.9254,
+      "step": 163
+    },
+    {
+      "epoch": 0.20410703173615433,
+      "grad_norm": 2.7914493083953857,
+      "learning_rate": 3.112033195020747e-06,
+      "loss": 1.9213,
+      "step": 164
+    },
+    {
+      "epoch": 0.20535158680771623,
+      "grad_norm": 3.3871428966522217,
+      "learning_rate": 3.132780082987552e-06,
+      "loss": 1.8993,
+      "step": 165
+    },
+    {
+      "epoch": 0.20659614187927816,
+      "grad_norm": 3.096653461456299,
+      "learning_rate": 3.153526970954357e-06,
+      "loss": 1.8827,
+      "step": 166
+    },
+    {
+      "epoch": 0.20784069695084006,
+      "grad_norm": 2.8276076316833496,
+      "learning_rate": 3.174273858921162e-06,
+      "loss": 1.912,
+      "step": 167
+    },
+    {
+      "epoch": 0.209085252022402,
+      "grad_norm": 3.3058435916900635,
+      "learning_rate": 3.1950207468879666e-06,
+      "loss": 1.8381,
+      "step": 168
+    },
+    {
+      "epoch": 0.2103298070939639,
+      "grad_norm": 3.5017333030700684,
+      "learning_rate": 3.215767634854772e-06,
+      "loss": 1.9544,
+      "step": 169
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "grad_norm": 3.457296133041382,
+      "learning_rate": 3.2365145228215773e-06,
+      "loss": 1.922,
+      "step": 170
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "eval_loss": 1.988455891609192,
+      "eval_runtime": 56.2099,
+      "eval_samples_per_second": 17.79,
+      "eval_steps_per_second": 0.747,
+      "step": 170
+    },
+    {
+      "epoch": 0.21281891723708773,
+      "grad_norm": 3.1964566707611084,
+      "learning_rate": 3.257261410788382e-06,
+      "loss": 1.8925,
+      "step": 171
+    },
+    {
+      "epoch": 0.21406347230864967,
+      "grad_norm": 3.234652042388916,
+      "learning_rate": 3.278008298755187e-06,
+      "loss": 1.884,
+      "step": 172
+    },
+    {
+      "epoch": 0.21530802738021157,
+      "grad_norm": 3.14414119720459,
+      "learning_rate": 3.2987551867219918e-06,
+      "loss": 1.8905,
+      "step": 173
+    },
+    {
+      "epoch": 0.2165525824517735,
+      "grad_norm": 3.606379508972168,
+      "learning_rate": 3.319502074688797e-06,
+      "loss": 1.946,
+      "step": 174
+    },
+    {
+      "epoch": 0.2177971375233354,
+      "grad_norm": 2.986646890640259,
+      "learning_rate": 3.3402489626556016e-06,
+      "loss": 1.8691,
+      "step": 175
+    },
+    {
+      "epoch": 0.21904169259489734,
+      "grad_norm": 3.7298269271850586,
+      "learning_rate": 3.360995850622407e-06,
+      "loss": 1.8717,
+      "step": 176
+    },
+    {
+      "epoch": 0.22028624766645924,
+      "grad_norm": 3.422295093536377,
+      "learning_rate": 3.381742738589212e-06,
+      "loss": 1.8568,
+      "step": 177
+    },
+    {
+      "epoch": 0.22153080273802117,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.8471,
+      "step": 178
+    },
+    {
+      "epoch": 0.22277535780958307,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.9303,
+      "step": 179
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "grad_norm": 3.506223440170288,
+      "learning_rate": 3.423236514522822e-06,
+      "loss": 1.8237,
+      "step": 180
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "eval_loss": 1.9734643697738647,
+      "eval_runtime": 52.2815,
+      "eval_samples_per_second": 19.127,
+      "eval_steps_per_second": 0.803,
+      "step": 180
+    },
+    {
+      "epoch": 0.2252644679527069,
+      "grad_norm": 3.0156619548797607,
+      "learning_rate": 3.4439834024896267e-06,
+      "loss": 1.908,
+      "step": 181
+    },
+    {
+      "epoch": 0.2265090230242688,
+      "grad_norm": 3.55517315864563,
+      "learning_rate": 3.4647302904564318e-06,
+      "loss": 1.9104,
+      "step": 182
+    },
+    {
+      "epoch": 0.22775357809583074,
+      "grad_norm": 3.144984006881714,
+      "learning_rate": 3.4854771784232365e-06,
+      "loss": 1.8198,
+      "step": 183
+    },
+    {
+      "epoch": 0.22899813316739265,
+      "grad_norm": 4.302074432373047,
+      "learning_rate": 3.5062240663900416e-06,
+      "loss": 1.8237,
+      "step": 184
+    },
+    {
+      "epoch": 0.23024268823895458,
+      "grad_norm": 3.0522446632385254,
+      "learning_rate": 3.526970954356847e-06,
+      "loss": 1.8409,
+      "step": 185
+    },
+    {
+      "epoch": 0.23148724331051648,
+      "grad_norm": 3.8607394695281982,
+      "learning_rate": 3.5477178423236518e-06,
+      "loss": 1.8807,
+      "step": 186
+    },
+    {
+      "epoch": 0.23273179838207841,
+      "grad_norm": 2.9236302375793457,
+      "learning_rate": 3.568464730290457e-06,
+      "loss": 1.7873,
+      "step": 187
+    },
+    {
+      "epoch": 0.23397635345364032,
+      "grad_norm": 4.013780117034912,
+      "learning_rate": 3.5892116182572616e-06,
+      "loss": 1.7909,
+      "step": 188
+    },
+    {
+      "epoch": 0.23522090852520225,
+      "grad_norm": 3.0933122634887695,
+      "learning_rate": 3.6099585062240667e-06,
+      "loss": 1.8469,
+      "step": 189
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "grad_norm": 3.487816095352173,
+      "learning_rate": 3.6307053941908714e-06,
+      "loss": 1.8469,
+      "step": 190
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "eval_loss": 1.9604240655899048,
+      "eval_runtime": 53.4604,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 0.786,
+      "step": 190
+    },
+    {
+      "epoch": 0.23771001866832608,
+      "grad_norm": 2.829219341278076,
+      "learning_rate": 3.6514522821576765e-06,
+      "loss": 1.8639,
+      "step": 191
+    },
+    {
+      "epoch": 0.238954573739888,
+      "grad_norm": 3.595534324645996,
+      "learning_rate": 3.672199170124482e-06,
+      "loss": 1.8988,
+      "step": 192
+    },
+    {
+      "epoch": 0.24019912881144992,
+      "grad_norm": 2.9022483825683594,
+      "learning_rate": 3.6929460580912867e-06,
+      "loss": 1.8972,
+      "step": 193
+    },
+    {
+      "epoch": 0.24144368388301182,
+      "grad_norm": 3.0667994022369385,
+      "learning_rate": 3.713692946058092e-06,
+      "loss": 1.8499,
+      "step": 194
+    },
+    {
+      "epoch": 0.24268823895457373,
+      "grad_norm": 3.0498485565185547,
+      "learning_rate": 3.7344398340248965e-06,
+      "loss": 1.8341,
+      "step": 195
+    },
+    {
+      "epoch": 0.24393279402613566,
+      "grad_norm": 4.058084964752197,
+      "learning_rate": 3.7551867219917016e-06,
+      "loss": 1.8641,
+      "step": 196
+    },
+    {
+      "epoch": 0.24517734909769756,
+      "grad_norm": 2.9995028972625732,
+      "learning_rate": 3.7759336099585063e-06,
+      "loss": 1.7885,
+      "step": 197
+    },
+    {
+      "epoch": 0.2464219041692595,
+      "grad_norm": 3.254704475402832,
+      "learning_rate": 3.7966804979253114e-06,
+      "loss": 1.7544,
+      "step": 198
+    },
+    {
+      "epoch": 0.2476664592408214,
+      "grad_norm": 2.9126744270324707,
+      "learning_rate": 3.817427385892116e-06,
+      "loss": 1.8461,
+      "step": 199
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "grad_norm": 3.475247621536255,
+      "learning_rate": 3.838174273858922e-06,
+      "loss": 1.8095,
+      "step": 200
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "eval_loss": 1.9373760223388672,
+      "eval_runtime": 51.8733,
+      "eval_samples_per_second": 19.278,
+      "eval_steps_per_second": 0.81,
+      "step": 200
+    },
+    {
+      "epoch": 0.25015556938394523,
+      "grad_norm": 3.2663731575012207,
+      "learning_rate": 3.858921161825726e-06,
+      "loss": 1.8076,
+      "step": 201
+    },
+    {
+      "epoch": 0.25140012445550713,
+      "grad_norm": 3.1151726245880127,
+      "learning_rate": 3.879668049792531e-06,
+      "loss": 1.8143,
+      "step": 202
+    },
+    {
+      "epoch": 0.2526446795270691,
+      "grad_norm": 3.102038860321045,
+      "learning_rate": 3.9004149377593365e-06,
+      "loss": 1.8279,
+      "step": 203
+    },
+    {
+      "epoch": 0.253889234598631,
+      "grad_norm": 3.370642900466919,
+      "learning_rate": 3.921161825726142e-06,
+      "loss": 1.8522,
+      "step": 204
+    },
+    {
+      "epoch": 0.2551337896701929,
+      "grad_norm": 3.116128921508789,
+      "learning_rate": 3.941908713692946e-06,
+      "loss": 1.8104,
+      "step": 205
+    },
+    {
+      "epoch": 0.2563783447417548,
+      "grad_norm": 3.12028431892395,
+      "learning_rate": 3.962655601659751e-06,
+      "loss": 1.7723,
+      "step": 206
+    },
+    {
+      "epoch": 0.25762289981331676,
+      "grad_norm": 3.1251418590545654,
+      "learning_rate": 3.983402489626556e-06,
+      "loss": 1.863,
+      "step": 207
+    },
+    {
+      "epoch": 0.25886745488487867,
+      "grad_norm": 3.073702812194824,
+      "learning_rate": 4.004149377593361e-06,
+      "loss": 1.805,
+      "step": 208
+    },
+    {
+      "epoch": 0.26011200995644057,
+      "grad_norm": 2.9772469997406006,
+      "learning_rate": 4.024896265560166e-06,
+      "loss": 1.8251,
+      "step": 209
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "grad_norm": 2.9970037937164307,
+      "learning_rate": 4.045643153526971e-06,
+      "loss": 1.8386,
+      "step": 210
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "eval_loss": 1.9167065620422363,
+      "eval_runtime": 42.3328,
+      "eval_samples_per_second": 23.622,
+      "eval_steps_per_second": 0.992,
+      "step": 210
+    },
+    {
+      "epoch": 0.26260112009956443,
+      "grad_norm": 3.1391561031341553,
+      "learning_rate": 4.0663900414937765e-06,
+      "loss": 1.8109,
+      "step": 211
+    },
+    {
+      "epoch": 0.26384567517112634,
+      "grad_norm": 3.161538600921631,
+      "learning_rate": 4.087136929460581e-06,
+      "loss": 1.8441,
+      "step": 212
+    },
+    {
+      "epoch": 0.26509023024268824,
+      "grad_norm": 3.12412166595459,
+      "learning_rate": 4.107883817427386e-06,
+      "loss": 1.7736,
+      "step": 213
+    },
+    {
+      "epoch": 0.26633478531425014,
+      "grad_norm": 3.3241145610809326,
+      "learning_rate": 4.128630705394191e-06,
+      "loss": 1.8067,
+      "step": 214
+    },
+    {
+      "epoch": 0.26757934038581205,
+      "grad_norm": 3.503307819366455,
+      "learning_rate": 4.149377593360996e-06,
+      "loss": 1.7858,
+      "step": 215
+    },
+    {
+      "epoch": 0.268823895457374,
+      "grad_norm": 3.1009578704833984,
+      "learning_rate": 4.170124481327801e-06,
+      "loss": 1.7542,
+      "step": 216
+    },
+    {
+      "epoch": 0.2700684505289359,
+      "grad_norm": 3.2236011028289795,
+      "learning_rate": 4.190871369294606e-06,
+      "loss": 1.8006,
+      "step": 217
+    },
+    {
+      "epoch": 0.2713130056004978,
+      "grad_norm": 3.048935651779175,
+      "learning_rate": 4.211618257261411e-06,
+      "loss": 1.7706,
+      "step": 218
+    },
+    {
+      "epoch": 0.2725575606720597,
+      "grad_norm": 3.3416147232055664,
+      "learning_rate": 4.232365145228216e-06,
+      "loss": 1.8108,
+      "step": 219
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "grad_norm": 3.2010738849639893,
+      "learning_rate": 4.253112033195021e-06,
+      "loss": 1.8561,
+      "step": 220
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "eval_loss": 1.9111930131912231,
+      "eval_runtime": 50.4077,
+      "eval_samples_per_second": 19.838,
+      "eval_steps_per_second": 0.833,
+      "step": 220
+    },
+    {
+      "epoch": 0.2750466708151836,
+      "grad_norm": 3.4033374786376953,
+      "learning_rate": 4.273858921161826e-06,
+      "loss": 1.8218,
+      "step": 221
+    },
+    {
+      "epoch": 0.2762912258867455,
+      "grad_norm": 3.2086002826690674,
+      "learning_rate": 4.294605809128631e-06,
+      "loss": 1.8213,
+      "step": 222
+    },
+    {
+      "epoch": 0.2775357809583074,
+      "grad_norm": 4.090956211090088,
+      "learning_rate": 4.315352697095436e-06,
+      "loss": 1.8455,
+      "step": 223
+    },
+    {
+      "epoch": 0.27878033602986935,
+      "grad_norm": 3.1142985820770264,
+      "learning_rate": 4.336099585062241e-06,
+      "loss": 1.7737,
+      "step": 224
+    },
+    {
+      "epoch": 0.28002489110143125,
+      "grad_norm": 3.369669198989868,
+      "learning_rate": 4.356846473029046e-06,
+      "loss": 1.824,
+      "step": 225
+    },
+    {
+      "epoch": 0.28126944617299315,
+      "grad_norm": 3.165672779083252,
+      "learning_rate": 4.3775933609958506e-06,
+      "loss": 1.7768,
+      "step": 226
+    },
+    {
+      "epoch": 0.28251400124455506,
+      "grad_norm": 3.3343470096588135,
+      "learning_rate": 4.398340248962656e-06,
+      "loss": 1.7278,
+      "step": 227
+    },
+    {
+      "epoch": 0.28375855631611696,
+      "grad_norm": 3.2574994564056396,
+      "learning_rate": 4.419087136929461e-06,
+      "loss": 1.8439,
+      "step": 228
+    },
+    {
+      "epoch": 0.2850031113876789,
+      "grad_norm": 3.043928623199463,
+      "learning_rate": 4.439834024896266e-06,
+      "loss": 1.818,
+      "step": 229
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "grad_norm": 2.9703125953674316,
+      "learning_rate": 4.460580912863071e-06,
+      "loss": 1.773,
+      "step": 230
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "eval_loss": 1.898109793663025,
+      "eval_runtime": 45.6023,
+      "eval_samples_per_second": 21.929,
+      "eval_steps_per_second": 0.921,
+      "step": 230
+    },
+    {
+      "epoch": 0.2874922215308027,
+      "grad_norm": 2.9997220039367676,
+      "learning_rate": 4.481327800829876e-06,
+      "loss": 1.7727,
+      "step": 231
+    },
+    {
+      "epoch": 0.28873677660236463,
+      "grad_norm": 3.1358482837677,
+      "learning_rate": 4.502074688796681e-06,
+      "loss": 1.7764,
+      "step": 232
+    },
+    {
+      "epoch": 0.2899813316739266,
+      "grad_norm": 3.395747661590576,
+      "learning_rate": 4.5228215767634855e-06,
+      "loss": 1.7826,
+      "step": 233
+    },
+    {
+      "epoch": 0.2912258867454885,
+      "grad_norm": 3.7494754791259766,
+      "learning_rate": 4.543568464730291e-06,
+      "loss": 1.7181,
+      "step": 234
+    },
+    {
+      "epoch": 0.2924704418170504,
+      "grad_norm": 3.42293381690979,
+      "learning_rate": 4.564315352697096e-06,
+      "loss": 1.7773,
+      "step": 235
+    },
+    {
+      "epoch": 0.2937149968886123,
+      "grad_norm": 3.2524514198303223,
+      "learning_rate": 4.585062240663901e-06,
+      "loss": 1.7294,
+      "step": 236
+    },
+    {
+      "epoch": 0.29495955196017426,
+      "grad_norm": 3.657869577407837,
+      "learning_rate": 4.605809128630706e-06,
+      "loss": 1.762,
+      "step": 237
+    },
+    {
+      "epoch": 0.29620410703173616,
+      "grad_norm": 3.127372980117798,
+      "learning_rate": 4.626556016597511e-06,
+      "loss": 1.7623,
+      "step": 238
+    },
+    {
+      "epoch": 0.29744866210329807,
+      "grad_norm": 4.166962146759033,
+      "learning_rate": 4.647302904564316e-06,
+      "loss": 1.6995,
+      "step": 239
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "grad_norm": 3.094264030456543,
+      "learning_rate": 4.66804979253112e-06,
+      "loss": 1.7469,
+      "step": 240
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "eval_loss": 1.8908178806304932,
+      "eval_runtime": 44.6332,
+      "eval_samples_per_second": 22.405,
+      "eval_steps_per_second": 0.941,
+      "step": 240
+    },
+    {
+      "epoch": 0.29993777224642193,
+      "grad_norm": 3.8644745349884033,
+      "learning_rate": 4.6887966804979255e-06,
+      "loss": 1.7644,
+      "step": 241
+    },
+    {
+      "epoch": 0.30118232731798383,
+      "grad_norm": 3.1488852500915527,
+      "learning_rate": 4.709543568464731e-06,
+      "loss": 1.772,
+      "step": 242
+    },
+    {
+      "epoch": 0.30242688238954574,
+      "grad_norm": 3.3179638385772705,
+      "learning_rate": 4.730290456431536e-06,
+      "loss": 1.7588,
+      "step": 243
+    },
+    {
+      "epoch": 0.30367143746110764,
+      "grad_norm": 3.328355550765991,
+      "learning_rate": 4.751037344398341e-06,
+      "loss": 1.7417,
+      "step": 244
+    },
+    {
+      "epoch": 0.30491599253266954,
+      "grad_norm": 3.2690482139587402,
+      "learning_rate": 4.771784232365146e-06,
+      "loss": 1.7562,
+      "step": 245
+    },
+    {
+      "epoch": 0.3061605476042315,
+      "grad_norm": 3.2759454250335693,
+      "learning_rate": 4.792531120331951e-06,
+      "loss": 1.7536,
+      "step": 246
+    },
+    {
+      "epoch": 0.3074051026757934,
+      "grad_norm": 2.937964916229248,
+      "learning_rate": 4.813278008298755e-06,
+      "loss": 1.7518,
+      "step": 247
+    },
+    {
+      "epoch": 0.3086496577473553,
+      "grad_norm": 3.0995302200317383,
+      "learning_rate": 4.83402489626556e-06,
+      "loss": 1.7593,
+      "step": 248
+    },
+    {
+      "epoch": 0.3098942128189172,
+      "grad_norm": 3.230459213256836,
+      "learning_rate": 4.8547717842323655e-06,
+      "loss": 1.6962,
+      "step": 249
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "grad_norm": 3.242576837539673,
+      "learning_rate": 4.875518672199171e-06,
+      "loss": 1.7307,
+      "step": 250
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "eval_loss": 1.8805371522903442,
+      "eval_runtime": 45.9886,
+      "eval_samples_per_second": 21.745,
+      "eval_steps_per_second": 0.913,
+      "step": 250
+    },
+    {
+      "epoch": 0.3123833229620411,
+      "grad_norm": 3.097045421600342,
+      "learning_rate": 4.896265560165976e-06,
+      "loss": 1.7438,
+      "step": 251
+    },
+    {
+      "epoch": 0.313627878033603,
+      "grad_norm": 3.2428948879241943,
+      "learning_rate": 4.91701244813278e-06,
+      "loss": 1.7179,
+      "step": 252
+    },
+    {
+      "epoch": 0.3148724331051649,
+      "grad_norm": 3.196274518966675,
+      "learning_rate": 4.937759336099586e-06,
+      "loss": 1.7855,
+      "step": 253
+    },
+    {
+      "epoch": 0.31611698817672684,
+      "grad_norm": 2.978203535079956,
+      "learning_rate": 4.95850622406639e-06,
+      "loss": 1.7144,
+      "step": 254
+    },
+    {
+      "epoch": 0.31736154324828875,
+      "grad_norm": 3.2641701698303223,
+      "learning_rate": 4.979253112033195e-06,
+      "loss": 1.7427,
+      "step": 255
+    },
+    {
+      "epoch": 0.31860609831985065,
+      "grad_norm": 2.7441232204437256,
+      "learning_rate": 5e-06,
+      "loss": 1.7619,
+      "step": 256
+    },
+    {
+      "epoch": 0.31985065339141255,
+      "grad_norm": 3.0723495483398438,
+      "learning_rate": 4.99769372693727e-06,
+      "loss": 1.8017,
+      "step": 257
+    },
+    {
+      "epoch": 0.3210952084629745,
+      "grad_norm": 3.104752540588379,
+      "learning_rate": 4.995387453874539e-06,
+      "loss": 1.7761,
+      "step": 258
+    },
+    {
+      "epoch": 0.3223397635345364,
+      "grad_norm": 3.138627052307129,
+      "learning_rate": 4.993081180811809e-06,
+      "loss": 1.7879,
+      "step": 259
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "grad_norm": 2.8567333221435547,
+      "learning_rate": 4.990774907749078e-06,
+      "loss": 1.7561,
+      "step": 260
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "eval_loss": 1.8606494665145874,
+      "eval_runtime": 46.453,
+      "eval_samples_per_second": 21.527,
+      "eval_steps_per_second": 0.904,
+      "step": 260
+    },
+    {
+      "epoch": 0.3248288736776602,
+      "grad_norm": 3.2696540355682373,
+      "learning_rate": 4.988468634686347e-06,
+      "loss": 1.7201,
+      "step": 261
+    },
+    {
+      "epoch": 0.3260734287492221,
+      "grad_norm": 2.6692731380462646,
+      "learning_rate": 4.986162361623617e-06,
+      "loss": 1.7264,
+      "step": 262
+    },
+    {
+      "epoch": 0.3273179838207841,
+      "grad_norm": 3.047549247741699,
+      "learning_rate": 4.983856088560886e-06,
+      "loss": 1.7362,
+      "step": 263
+    },
+    {
+      "epoch": 0.328562538892346,
+      "grad_norm": 3.0667457580566406,
+      "learning_rate": 4.981549815498156e-06,
+      "loss": 1.7321,
+      "step": 264
+    },
+    {
+      "epoch": 0.3298070939639079,
+      "grad_norm": 2.560047149658203,
+      "learning_rate": 4.979243542435424e-06,
+      "loss": 1.7508,
+      "step": 265
+    },
+    {
+      "epoch": 0.3310516490354698,
+      "grad_norm": 3.26595401763916,
+      "learning_rate": 4.976937269372694e-06,
+      "loss": 1.7248,
+      "step": 266
+    },
+    {
+      "epoch": 0.33229620410703176,
+      "grad_norm": 2.929210662841797,
+      "learning_rate": 4.974630996309964e-06,
+      "loss": 1.697,
+      "step": 267
+    },
+    {
+      "epoch": 0.33354075917859366,
+      "grad_norm": 3.1526286602020264,
+      "learning_rate": 4.972324723247233e-06,
+      "loss": 1.7385,
+      "step": 268
+    },
+    {
+      "epoch": 0.33478531425015556,
+      "grad_norm": 3.001619577407837,
+      "learning_rate": 4.970018450184502e-06,
+      "loss": 1.7072,
+      "step": 269
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "grad_norm": 3.2464189529418945,
+      "learning_rate": 4.9677121771217715e-06,
+      "loss": 1.7361,
+      "step": 270
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "eval_loss": 1.860226035118103,
+      "eval_runtime": 46.6881,
+      "eval_samples_per_second": 21.419,
+      "eval_steps_per_second": 0.9,
+      "step": 270
+    },
+    {
+      "epoch": 0.3372744243932794,
+      "grad_norm": 3.2104530334472656,
+      "learning_rate": 4.965405904059041e-06,
+      "loss": 1.7171,
+      "step": 271
+    },
+    {
+      "epoch": 0.33851897946484133,
+      "grad_norm": 3.146847724914551,
+      "learning_rate": 4.96309963099631e-06,
+      "loss": 1.7311,
+      "step": 272
+    },
+    {
+      "epoch": 0.33976353453640323,
+      "grad_norm": 3.1431286334991455,
+      "learning_rate": 4.96079335793358e-06,
+      "loss": 1.7345,
+      "step": 273
+    },
+    {
+      "epoch": 0.34100808960796514,
+      "grad_norm": 2.847163677215576,
+      "learning_rate": 4.958487084870849e-06,
+      "loss": 1.7262,
+      "step": 274
+    },
+    {
+      "epoch": 0.3422526446795271,
+      "grad_norm": 3.1987810134887695,
+      "learning_rate": 4.956180811808119e-06,
+      "loss": 1.7945,
+      "step": 275
+    },
+    {
+      "epoch": 0.343497199751089,
+      "grad_norm": 2.7475385665893555,
+      "learning_rate": 4.953874538745388e-06,
+      "loss": 1.76,
+      "step": 276
+    },
+    {
+      "epoch": 0.3447417548226509,
+      "grad_norm": 3.0427663326263428,
+      "learning_rate": 4.9515682656826574e-06,
+      "loss": 1.7324,
+      "step": 277
+    },
+    {
+      "epoch": 0.3459863098942128,
+      "grad_norm": 3.605212688446045,
+      "learning_rate": 4.949261992619927e-06,
+      "loss": 1.7712,
+      "step": 278
+    },
+    {
+      "epoch": 0.3472308649657747,
+      "grad_norm": 3.0564935207366943,
+      "learning_rate": 4.946955719557196e-06,
+      "loss": 1.6792,
+      "step": 279
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "grad_norm": 3.1069741249084473,
+      "learning_rate": 4.944649446494466e-06,
+      "loss": 1.7145,
+      "step": 280
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "eval_loss": 1.8537051677703857,
+      "eval_runtime": 47.4813,
+      "eval_samples_per_second": 21.061,
+      "eval_steps_per_second": 0.885,
+      "step": 280
+    },
+    {
+      "epoch": 0.3497199751088986,
+      "grad_norm": 2.8021512031555176,
+      "learning_rate": 4.942343173431734e-06,
+      "loss": 1.7145,
+      "step": 281
+    },
+    {
+      "epoch": 0.3509645301804605,
+      "grad_norm": 3.5549023151397705,
+      "learning_rate": 4.940036900369004e-06,
+      "loss": 1.7642,
+      "step": 282
+    },
+    {
+      "epoch": 0.3522090852520224,
+      "grad_norm": 2.7648985385894775,
+      "learning_rate": 4.937730627306274e-06,
+      "loss": 1.7255,
+      "step": 283
+    },
+    {
+      "epoch": 0.35345364032358434,
+      "grad_norm": 3.0815863609313965,
+      "learning_rate": 4.9354243542435426e-06,
+      "loss": 1.7055,
+      "step": 284
+    },
+    {
+      "epoch": 0.35469819539514624,
+      "grad_norm": 2.9009227752685547,
+      "learning_rate": 4.933118081180812e-06,
+      "loss": 1.7019,
+      "step": 285
+    },
+    {
+      "epoch": 0.35594275046670815,
+      "grad_norm": 2.618429660797119,
+      "learning_rate": 4.930811808118081e-06,
+      "loss": 1.6318,
+      "step": 286
+    },
+    {
+      "epoch": 0.35718730553827005,
+      "grad_norm": 3.368230104446411,
+      "learning_rate": 4.928505535055351e-06,
+      "loss": 1.7304,
+      "step": 287
+    },
+    {
+      "epoch": 0.358431860609832,
+      "grad_norm": 3.256889820098877,
+      "learning_rate": 4.92619926199262e-06,
+      "loss": 1.7019,
+      "step": 288
+    },
+    {
+      "epoch": 0.3596764156813939,
+      "grad_norm": 2.9366304874420166,
+      "learning_rate": 4.92389298892989e-06,
+      "loss": 1.734,
+      "step": 289
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "grad_norm": 3.0393142700195312,
+      "learning_rate": 4.921586715867159e-06,
+      "loss": 1.7504,
+      "step": 290
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "eval_loss": 1.8359886407852173,
+      "eval_runtime": 49.7622,
+      "eval_samples_per_second": 20.096,
+      "eval_steps_per_second": 0.844,
+      "step": 290
+    },
+    {
+      "epoch": 0.3621655258245177,
+      "grad_norm": 2.7288589477539062,
+      "learning_rate": 4.9192804428044285e-06,
+      "loss": 1.7004,
+      "step": 291
+    },
+    {
+      "epoch": 0.3634100808960797,
+      "grad_norm": 3.2793378829956055,
+      "learning_rate": 4.916974169741698e-06,
+      "loss": 1.7028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3646546359676416,
+      "grad_norm": 2.960880756378174,
+      "learning_rate": 4.914667896678967e-06,
+      "loss": 1.6759,
+      "step": 293
+    },
+    {
+      "epoch": 0.3658991910392035,
+      "grad_norm": 2.836421012878418,
+      "learning_rate": 4.912361623616237e-06,
+      "loss": 1.6475,
+      "step": 294
+    },
+    {
+      "epoch": 0.3671437461107654,
+      "grad_norm": 2.9495465755462646,
+      "learning_rate": 4.910055350553506e-06,
+      "loss": 1.7076,
+      "step": 295
+    },
+    {
+      "epoch": 0.3683883011823273,
+      "grad_norm": 2.954730272293091,
+      "learning_rate": 4.907749077490776e-06,
+      "loss": 1.7261,
+      "step": 296
+    },
+    {
+      "epoch": 0.36963285625388925,
+      "grad_norm": 2.8167543411254883,
+      "learning_rate": 4.905442804428044e-06,
+      "loss": 1.6939,
+      "step": 297
+    },
+    {
+      "epoch": 0.37087741132545116,
+      "grad_norm": 2.857316017150879,
+      "learning_rate": 4.903136531365314e-06,
+      "loss": 1.7362,
+      "step": 298
+    },
+    {
+      "epoch": 0.37212196639701306,
+      "grad_norm": 2.8612918853759766,
+      "learning_rate": 4.900830258302584e-06,
+      "loss": 1.6489,
+      "step": 299
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "grad_norm": 2.9618914127349854,
+      "learning_rate": 4.898523985239853e-06,
+      "loss": 1.718,
+      "step": 300
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "eval_loss": 1.827571988105774,
+      "eval_runtime": 46.4552,
+      "eval_samples_per_second": 21.526,
+      "eval_steps_per_second": 0.904,
+      "step": 300
+    },
+    {
+      "epoch": 0.3746110765401369,
+      "grad_norm": 2.8706953525543213,
+      "learning_rate": 4.896217712177122e-06,
+      "loss": 1.6925,
+      "step": 301
+    },
+    {
+      "epoch": 0.3758556316116988,
+      "grad_norm": 3.2565090656280518,
+      "learning_rate": 4.893911439114391e-06,
+      "loss": 1.7369,
+      "step": 302
+    },
+    {
+      "epoch": 0.37710018668326073,
+      "grad_norm": 2.928858995437622,
+      "learning_rate": 4.891605166051661e-06,
+      "loss": 1.6882,
+      "step": 303
+    },
+    {
+      "epoch": 0.37834474175482263,
+      "grad_norm": 3.042314052581787,
+      "learning_rate": 4.88929889298893e-06,
+      "loss": 1.6466,
+      "step": 304
+    },
+    {
+      "epoch": 0.3795892968263846,
+      "grad_norm": 2.7509143352508545,
+      "learning_rate": 4.8869926199262e-06,
+      "loss": 1.685,
+      "step": 305
+    },
+    {
+      "epoch": 0.3808338518979465,
+      "grad_norm": 2.9601588249206543,
+      "learning_rate": 4.884686346863469e-06,
+      "loss": 1.6772,
+      "step": 306
+    },
+    {
+      "epoch": 0.3820784069695084,
+      "grad_norm": 2.618608236312866,
+      "learning_rate": 4.8823800738007384e-06,
+      "loss": 1.7127,
+      "step": 307
+    },
+    {
+      "epoch": 0.3833229620410703,
+      "grad_norm": 3.047581195831299,
+      "learning_rate": 4.880073800738008e-06,
+      "loss": 1.7174,
+      "step": 308
+    },
+    {
+      "epoch": 0.3845675171126322,
+      "grad_norm": 2.87735652923584,
+      "learning_rate": 4.877767527675277e-06,
+      "loss": 1.6677,
+      "step": 309
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "grad_norm": 2.7169721126556396,
+      "learning_rate": 4.875461254612546e-06,
+      "loss": 1.7486,
+      "step": 310
+    },
+    {
+      "epoch": 0.38581207218419417,
+      "eval_loss": 1.8221794366836548,
+      "eval_runtime": 43.1681,
+      "eval_samples_per_second": 23.165,
+      "eval_steps_per_second": 0.973,
+      "step": 310
+    },
+    {
+      "epoch": 0.38705662725575607,
+      "grad_norm": 2.7094991207122803,
+      "learning_rate": 4.873154981549816e-06,
+      "loss": 1.7348,
+      "step": 311
+    },
+    {
+      "epoch": 0.388301182327318,
+      "grad_norm": 2.8989078998565674,
+      "learning_rate": 4.8708487084870856e-06,
+      "loss": 1.7352,
+      "step": 312
+    },
+    {
+      "epoch": 0.3895457373988799,
+      "grad_norm": 2.9565601348876953,
+      "learning_rate": 4.868542435424355e-06,
+      "loss": 1.7063,
+      "step": 313
+    },
+    {
+      "epoch": 0.39079029247044184,
+      "grad_norm": 2.8354082107543945,
+      "learning_rate": 4.8662361623616235e-06,
+      "loss": 1.6808,
+      "step": 314
+    },
+    {
+      "epoch": 0.39203484754200374,
+      "grad_norm": 2.8287479877471924,
+      "learning_rate": 4.863929889298894e-06,
+      "loss": 1.6731,
+      "step": 315
+    },
+    {
+      "epoch": 0.39327940261356564,
+      "grad_norm": 2.6537821292877197,
+      "learning_rate": 4.861623616236163e-06,
+      "loss": 1.695,
+      "step": 316
+    },
+    {
+      "epoch": 0.39452395768512755,
+      "grad_norm": 2.7135190963745117,
+      "learning_rate": 4.859317343173432e-06,
+      "loss": 1.6897,
+      "step": 317
+    },
+    {
+      "epoch": 0.3957685127566895,
+      "grad_norm": 3.1019129753112793,
+      "learning_rate": 4.857011070110701e-06,
+      "loss": 1.7219,
+      "step": 318
+    },
+    {
+      "epoch": 0.3970130678282514,
+      "grad_norm": 2.945737838745117,
+      "learning_rate": 4.854704797047971e-06,
+      "loss": 1.7316,
+      "step": 319
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "grad_norm": 2.7668251991271973,
+      "learning_rate": 4.85239852398524e-06,
+      "loss": 1.6461,
+      "step": 320
+    },
+    {
+      "epoch": 0.3982576228998133,
+      "eval_loss": 1.8210570812225342,
+      "eval_runtime": 46.2552,
+      "eval_samples_per_second": 21.619,
+      "eval_steps_per_second": 0.908,
+      "step": 320
+    },
+    {
+      "epoch": 0.3995021779713752,
+      "grad_norm": 2.738109588623047,
+      "learning_rate": 4.8500922509225095e-06,
+      "loss": 1.6907,
+      "step": 321
+    },
+    {
+      "epoch": 0.4007467330429372,
+      "grad_norm": 3.0195062160491943,
+      "learning_rate": 4.847785977859779e-06,
+      "loss": 1.7099,
+      "step": 322
+    },
+    {
+      "epoch": 0.4019912881144991,
+      "grad_norm": 2.8672404289245605,
+      "learning_rate": 4.845479704797048e-06,
+      "loss": 1.6509,
+      "step": 323
+    },
+    {
+      "epoch": 0.403235843186061,
+      "grad_norm": 2.9041008949279785,
+      "learning_rate": 4.843173431734318e-06,
+      "loss": 1.5818,
+      "step": 324
+    },
+    {
+      "epoch": 0.4044803982576229,
+      "grad_norm": 2.9105985164642334,
+      "learning_rate": 4.840867158671587e-06,
+      "loss": 1.7002,
+      "step": 325
+    },
+    {
+      "epoch": 0.4057249533291848,
+      "grad_norm": 2.749009847640991,
+      "learning_rate": 4.838560885608857e-06,
+      "loss": 1.7021,
+      "step": 326
+    },
+    {
+      "epoch": 0.40696950840074675,
+      "grad_norm": 2.6998822689056396,
+      "learning_rate": 4.836254612546126e-06,
+      "loss": 1.6853,
+      "step": 327
+    },
+    {
+      "epoch": 0.40821406347230865,
+      "grad_norm": 2.678201675415039,
+      "learning_rate": 4.8339483394833955e-06,
+      "loss": 1.6934,
+      "step": 328
+    },
+    {
+      "epoch": 0.40945861854387056,
+      "grad_norm": 2.774845600128174,
+      "learning_rate": 4.831642066420665e-06,
+      "loss": 1.657,
+      "step": 329
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "grad_norm": 2.6363000869750977,
+      "learning_rate": 4.8293357933579335e-06,
+      "loss": 1.6555,
+      "step": 330
+    },
+    {
+      "epoch": 0.41070317361543246,
+      "eval_loss": 1.8126581907272339,
+      "eval_runtime": 52.7717,
+      "eval_samples_per_second": 18.95,
+      "eval_steps_per_second": 0.796,
+      "step": 330
+    },
+    {
+      "epoch": 0.4119477286869944,
+      "grad_norm": 3.011659622192383,
+      "learning_rate": 4.827029520295204e-06,
+      "loss": 1.7202,
+      "step": 331
+    },
+    {
+      "epoch": 0.4131922837585563,
+      "grad_norm": 2.8322594165802,
+      "learning_rate": 4.824723247232473e-06,
+      "loss": 1.6736,
+      "step": 332
+    },
+    {
+      "epoch": 0.4144368388301182,
+      "grad_norm": 2.9360146522521973,
+      "learning_rate": 4.822416974169742e-06,
+      "loss": 1.7213,
+      "step": 333
+    },
+    {
+      "epoch": 0.41568139390168013,
+      "grad_norm": 2.810206413269043,
+      "learning_rate": 4.820110701107011e-06,
+      "loss": 1.6867,
+      "step": 334
+    },
+    {
+      "epoch": 0.4169259489732421,
+      "grad_norm": 2.6864097118377686,
+      "learning_rate": 4.817804428044281e-06,
+      "loss": 1.6854,
+      "step": 335
+    },
+    {
+      "epoch": 0.418170504044804,
+      "grad_norm": 2.7396061420440674,
+      "learning_rate": 4.81549815498155e-06,
+      "loss": 1.68,
+      "step": 336
+    },
+    {
+      "epoch": 0.4194150591163659,
+      "grad_norm": 2.602771520614624,
+      "learning_rate": 4.8131918819188194e-06,
+      "loss": 1.6269,
+      "step": 337
+    },
+    {
+      "epoch": 0.4206596141879278,
+      "grad_norm": 2.8174757957458496,
+      "learning_rate": 4.810885608856089e-06,
+      "loss": 1.6981,
+      "step": 338
+    },
+    {
+      "epoch": 0.42190416925948976,
+      "grad_norm": 3.2398130893707275,
+      "learning_rate": 4.808579335793358e-06,
+      "loss": 1.7061,
+      "step": 339
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "grad_norm": 2.7119736671447754,
+      "learning_rate": 4.806273062730628e-06,
+      "loss": 1.6875,
+      "step": 340
+    },
+    {
+      "epoch": 0.42314872433105166,
+      "eval_loss": 1.7973068952560425,
+      "eval_runtime": 53.5791,
+      "eval_samples_per_second": 18.664,
+      "eval_steps_per_second": 0.784,
+      "step": 340
+    },
+    {
+      "epoch": 0.42439327940261357,
+      "grad_norm": 2.868533134460449,
+      "learning_rate": 4.803966789667897e-06,
+      "loss": 1.6967,
+      "step": 341
+    },
+    {
+      "epoch": 0.42563783447417547,
+      "grad_norm": 2.528083562850952,
+      "learning_rate": 4.8016605166051665e-06,
+      "loss": 1.6195,
+      "step": 342
+    },
+    {
+      "epoch": 0.4268823895457374,
+      "grad_norm": 2.7885115146636963,
+      "learning_rate": 4.799354243542436e-06,
+      "loss": 1.6197,
+      "step": 343
+    },
+    {
+      "epoch": 0.42812694461729933,
+      "grad_norm": 2.768247604370117,
+      "learning_rate": 4.797047970479705e-06,
+      "loss": 1.6271,
+      "step": 344
+    },
+    {
+      "epoch": 0.42937149968886124,
+      "grad_norm": 2.7710275650024414,
+      "learning_rate": 4.794741697416975e-06,
+      "loss": 1.6768,
+      "step": 345
+    },
+    {
+      "epoch": 0.43061605476042314,
+      "grad_norm": 2.968337297439575,
+      "learning_rate": 4.792435424354243e-06,
+      "loss": 1.6433,
+      "step": 346
+    },
+    {
+      "epoch": 0.43186060983198504,
+      "grad_norm": 2.5739808082580566,
+      "learning_rate": 4.790129151291514e-06,
+      "loss": 1.6327,
+      "step": 347
+    },
+    {
+      "epoch": 0.433105164903547,
+      "grad_norm": 2.7676408290863037,
+      "learning_rate": 4.787822878228783e-06,
+      "loss": 1.6178,
+      "step": 348
+    },
+    {
+      "epoch": 0.4343497199751089,
+      "grad_norm": 2.959059476852417,
+      "learning_rate": 4.7855166051660525e-06,
+      "loss": 1.6809,
+      "step": 349
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "grad_norm": 2.559967279434204,
+      "learning_rate": 4.783210332103321e-06,
+      "loss": 1.6704,
+      "step": 350
+    },
+    {
+      "epoch": 0.4355942750466708,
+      "eval_loss": 1.8013949394226074,
+      "eval_runtime": 50.4649,
+      "eval_samples_per_second": 19.816,
+      "eval_steps_per_second": 0.832,
+      "step": 350
+    },
+    {
+      "epoch": 0.4368388301182327,
+      "grad_norm": 2.817901611328125,
+      "learning_rate": 4.7809040590405905e-06,
+      "loss": 1.6296,
+      "step": 351
+    },
+    {
+      "epoch": 0.43808338518979467,
+      "grad_norm": 2.7116055488586426,
+      "learning_rate": 4.778597785977861e-06,
+      "loss": 1.6652,
+      "step": 352
+    },
+    {
+      "epoch": 0.4393279402613566,
+      "grad_norm": 2.544750452041626,
+      "learning_rate": 4.776291512915129e-06,
+      "loss": 1.5668,
+      "step": 353
+    },
+    {
+      "epoch": 0.4405724953329185,
+      "grad_norm": 2.720534086227417,
+      "learning_rate": 4.773985239852399e-06,
+      "loss": 1.6152,
+      "step": 354
+    },
+    {
+      "epoch": 0.4418170504044804,
+      "grad_norm": 2.7270126342773438,
+      "learning_rate": 4.771678966789668e-06,
+      "loss": 1.5624,
+      "step": 355
+    },
+    {
+      "epoch": 0.44306160547604234,
+      "grad_norm": 2.7986624240875244,
+      "learning_rate": 4.769372693726938e-06,
+      "loss": 1.6621,
+      "step": 356
+    },
+    {
+      "epoch": 0.44430616054760425,
+      "grad_norm": 2.943107843399048,
+      "learning_rate": 4.767066420664207e-06,
+      "loss": 1.6713,
+      "step": 357
+    },
+    {
+      "epoch": 0.44555071561916615,
+      "grad_norm": 2.739898681640625,
+      "learning_rate": 4.7647601476014765e-06,
+      "loss": 1.6363,
+      "step": 358
+    },
+    {
+      "epoch": 0.44679527069072805,
+      "grad_norm": 2.8729329109191895,
+      "learning_rate": 4.762453874538746e-06,
+      "loss": 1.6605,
+      "step": 359
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "grad_norm": 2.909029483795166,
+      "learning_rate": 4.760147601476015e-06,
+      "loss": 1.6277,
+      "step": 360
+    },
+    {
+      "epoch": 0.44803982576228996,
+      "eval_loss": 1.8110179901123047,
+      "eval_runtime": 47.924,
+      "eval_samples_per_second": 20.866,
+      "eval_steps_per_second": 0.876,
+      "step": 360
+    },
+    {
+      "epoch": 0.4492843808338519,
+      "grad_norm": 3.3826682567596436,
+      "learning_rate": 4.757841328413285e-06,
+      "loss": 1.7155,
+      "step": 361
+    },
+    {
+      "epoch": 0.4505289359054138,
+      "grad_norm": 3.0586729049682617,
+      "learning_rate": 4.755535055350554e-06,
+      "loss": 1.5991,
+      "step": 362
+    },
+    {
+      "epoch": 0.4517734909769757,
+      "grad_norm": 2.645576238632202,
+      "learning_rate": 4.753228782287823e-06,
+      "loss": 1.6504,
+      "step": 363
+    },
+    {
+      "epoch": 0.4530180460485376,
+      "grad_norm": 2.705967664718628,
+      "learning_rate": 4.750922509225093e-06,
+      "loss": 1.6761,
+      "step": 364
+    },
+    {
+      "epoch": 0.4542626011200996,
+      "grad_norm": 2.7651922702789307,
+      "learning_rate": 4.748616236162362e-06,
+      "loss": 1.61,
+      "step": 365
+    },
+    {
+      "epoch": 0.4555071561916615,
+      "grad_norm": 2.5654966831207275,
+      "learning_rate": 4.746309963099631e-06,
+      "loss": 1.6335,
+      "step": 366
+    },
+    {
+      "epoch": 0.4567517112632234,
+      "grad_norm": 2.6606173515319824,
+      "learning_rate": 4.7440036900369e-06,
+      "loss": 1.6207,
+      "step": 367
+    },
+    {
+      "epoch": 0.4579962663347853,
+      "grad_norm": 2.724755048751831,
+      "learning_rate": 4.741697416974171e-06,
+      "loss": 1.6329,
+      "step": 368
+    },
+    {
+      "epoch": 0.45924082140634725,
+      "grad_norm": 2.7019615173339844,
+      "learning_rate": 4.739391143911439e-06,
+      "loss": 1.6763,
+      "step": 369
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "grad_norm": 2.6594254970550537,
+      "learning_rate": 4.737084870848709e-06,
+      "loss": 1.6174,
+      "step": 370
+    },
+    {
+      "epoch": 0.46048537647790916,
+      "eval_loss": 1.7924141883850098,
+      "eval_runtime": 53.0349,
+      "eval_samples_per_second": 18.855,
+      "eval_steps_per_second": 0.792,
+      "step": 370
+    },
+    {
+      "epoch": 0.46172993154947106,
+      "grad_norm": 2.668421745300293,
+      "learning_rate": 4.734778597785978e-06,
+      "loss": 1.6421,
+      "step": 371
+    },
+    {
+      "epoch": 0.46297448662103297,
+      "grad_norm": 2.7148995399475098,
+      "learning_rate": 4.7324723247232475e-06,
+      "loss": 1.6635,
+      "step": 372
+    },
+    {
+      "epoch": 0.4642190416925949,
+      "grad_norm": 3.156646490097046,
+      "learning_rate": 4.730166051660517e-06,
+      "loss": 1.6679,
+      "step": 373
+    },
+    {
+      "epoch": 0.46546359676415683,
+      "grad_norm": 2.8170645236968994,
+      "learning_rate": 4.727859778597786e-06,
+      "loss": 1.6492,
+      "step": 374
+    },
+    {
+      "epoch": 0.46670815183571873,
+      "grad_norm": 2.9796457290649414,
+      "learning_rate": 4.725553505535056e-06,
+      "loss": 1.6025,
+      "step": 375
+    },
+    {
+      "epoch": 0.46795270690728064,
+      "grad_norm": 2.6493406295776367,
+      "learning_rate": 4.723247232472325e-06,
+      "loss": 1.6398,
+      "step": 376
+    },
+    {
+      "epoch": 0.46919726197884254,
+      "grad_norm": 3.2643542289733887,
+      "learning_rate": 4.720940959409595e-06,
+      "loss": 1.6413,
+      "step": 377
+    },
+    {
+      "epoch": 0.4704418170504045,
+      "grad_norm": 2.7613179683685303,
+      "learning_rate": 4.718634686346864e-06,
+      "loss": 1.6251,
+      "step": 378
+    },
+    {
+      "epoch": 0.4716863721219664,
+      "grad_norm": 3.1216518878936768,
+      "learning_rate": 4.716328413284133e-06,
+      "loss": 1.6268,
+      "step": 379
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "grad_norm": 2.571647882461548,
+      "learning_rate": 4.714022140221403e-06,
+      "loss": 1.6245,
+      "step": 380
+    },
+    {
+      "epoch": 0.4729309271935283,
+      "eval_loss": 1.7875893115997314,
+      "eval_runtime": 49.3058,
+      "eval_samples_per_second": 20.282,
+      "eval_steps_per_second": 0.852,
+      "step": 380
+    },
+    {
+      "epoch": 0.4741754822650902,
+      "grad_norm": 3.443906545639038,
+      "learning_rate": 4.711715867158672e-06,
+      "loss": 1.6799,
+      "step": 381
+    },
+    {
+      "epoch": 0.47542003733665217,
+      "grad_norm": 2.7887957096099854,
+      "learning_rate": 4.709409594095941e-06,
+      "loss": 1.6589,
+      "step": 382
+    },
+    {
+      "epoch": 0.47666459240821407,
+      "grad_norm": 3.2833151817321777,
+      "learning_rate": 4.70710332103321e-06,
+      "loss": 1.6064,
+      "step": 383
+    },
+    {
+      "epoch": 0.477909147479776,
+      "grad_norm": 2.6975057125091553,
+      "learning_rate": 4.704797047970481e-06,
+      "loss": 1.6029,
+      "step": 384
+    },
+    {
+      "epoch": 0.4791537025513379,
+      "grad_norm": 2.745246171951294,
+      "learning_rate": 4.702490774907749e-06,
+      "loss": 1.7127,
+      "step": 385
+    },
+    {
+      "epoch": 0.48039825762289984,
+      "grad_norm": 3.0522775650024414,
+      "learning_rate": 4.700184501845019e-06,
+      "loss": 1.6508,
+      "step": 386
+    },
+    {
+      "epoch": 0.48164281269446174,
+      "grad_norm": 2.689218044281006,
+      "learning_rate": 4.697878228782288e-06,
+      "loss": 1.6971,
+      "step": 387
+    },
+    {
+      "epoch": 0.48288736776602365,
+      "grad_norm": 3.060361862182617,
+      "learning_rate": 4.6955719557195575e-06,
+      "loss": 1.6464,
+      "step": 388
+    },
+    {
+      "epoch": 0.48413192283758555,
+      "grad_norm": 2.872262477874756,
+      "learning_rate": 4.693265682656827e-06,
+      "loss": 1.6293,
+      "step": 389
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "grad_norm": 2.980050563812256,
+      "learning_rate": 4.690959409594096e-06,
+      "loss": 1.6435,
+      "step": 390
+    },
+    {
+      "epoch": 0.48537647790914745,
+      "eval_loss": 1.7791496515274048,
+      "eval_runtime": 50.0299,
+      "eval_samples_per_second": 19.988,
+      "eval_steps_per_second": 0.839,
+      "step": 390
+    },
+    {
+      "epoch": 0.4866210329807094,
+      "grad_norm": 2.764827251434326,
+      "learning_rate": 4.688653136531366e-06,
+      "loss": 1.6279,
+      "step": 391
+    },
+    {
+      "epoch": 0.4878655880522713,
+      "grad_norm": 2.6912333965301514,
+      "learning_rate": 4.686346863468635e-06,
+      "loss": 1.7276,
+      "step": 392
+    },
+    {
+      "epoch": 0.4891101431238332,
+      "grad_norm": 2.8358330726623535,
+      "learning_rate": 4.6840405904059046e-06,
+      "loss": 1.6482,
+      "step": 393
+    },
+    {
+      "epoch": 0.4903546981953951,
+      "grad_norm": 2.6581575870513916,
+      "learning_rate": 4.681734317343174e-06,
+      "loss": 1.6853,
+      "step": 394
+    },
+    {
+      "epoch": 0.4915992532669571,
+      "grad_norm": 2.941425323486328,
+      "learning_rate": 4.6794280442804426e-06,
+      "loss": 1.6075,
+      "step": 395
+    },
+    {
+      "epoch": 0.492843808338519,
+      "grad_norm": 3.2334814071655273,
+      "learning_rate": 4.677121771217713e-06,
+      "loss": 1.6157,
+      "step": 396
+    },
+    {
+      "epoch": 0.4940883634100809,
+      "grad_norm": 2.984410524368286,
+      "learning_rate": 4.674815498154982e-06,
+      "loss": 1.6257,
+      "step": 397
+    },
+    {
+      "epoch": 0.4953329184816428,
+      "grad_norm": 3.0650789737701416,
+      "learning_rate": 4.672509225092252e-06,
+      "loss": 1.6573,
+      "step": 398
+    },
+    {
+      "epoch": 0.49657747355320475,
+      "grad_norm": 2.798664093017578,
+      "learning_rate": 4.67020295202952e-06,
+      "loss": 1.638,
+      "step": 399
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "grad_norm": 2.8266706466674805,
+      "learning_rate": 4.66789667896679e-06,
+      "loss": 1.6484,
+      "step": 400
+    },
+    {
+      "epoch": 0.49782202862476665,
+      "eval_loss": 1.7714862823486328,
+      "eval_runtime": 48.0657,
+      "eval_samples_per_second": 20.805,
+      "eval_steps_per_second": 0.874,
+      "step": 400
+    },
+    {
+      "epoch": 0.49906658369632856,
+      "grad_norm": 2.9928388595581055,
+      "learning_rate": 4.66559040590406e-06,
+      "loss": 1.6721,
+      "step": 401
+    },
+    {
+      "epoch": 0.5003111387678905,
+      "grad_norm": 2.573514461517334,
+      "learning_rate": 4.6632841328413285e-06,
+      "loss": 1.679,
+      "step": 402
+    },
+    {
+      "epoch": 0.5015556938394524,
+      "grad_norm": 3.1736996173858643,
+      "learning_rate": 4.660977859778598e-06,
+      "loss": 1.7124,
+      "step": 403
+    },
+    {
+      "epoch": 0.5028002489110143,
+      "grad_norm": 2.858849287033081,
+      "learning_rate": 4.658671586715867e-06,
+      "loss": 1.6622,
+      "step": 404
+    },
+    {
+      "epoch": 0.5040448039825762,
+      "grad_norm": 2.825698137283325,
+      "learning_rate": 4.656365313653137e-06,
+      "loss": 1.6464,
+      "step": 405
+    },
+    {
+      "epoch": 0.5052893590541382,
+      "grad_norm": 3.783891439437866,
+      "learning_rate": 4.654059040590406e-06,
+      "loss": 1.6665,
+      "step": 406
+    },
+    {
+      "epoch": 0.5065339141257,
+      "grad_norm": 2.789813756942749,
+      "learning_rate": 4.651752767527676e-06,
+      "loss": 1.6064,
+      "step": 407
+    },
+    {
+      "epoch": 0.507778469197262,
+      "grad_norm": 3.3106348514556885,
+      "learning_rate": 4.649446494464945e-06,
+      "loss": 1.6542,
+      "step": 408
+    },
+    {
+      "epoch": 0.5090230242688238,
+      "grad_norm": 2.8805112838745117,
+      "learning_rate": 4.6471402214022145e-06,
+      "loss": 1.6058,
+      "step": 409
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "grad_norm": 3.30898118019104,
+      "learning_rate": 4.644833948339484e-06,
+      "loss": 1.5933,
+      "step": 410
+    },
+    {
+      "epoch": 0.5102675793403858,
+      "eval_loss": 1.7787818908691406,
+      "eval_runtime": 44.0876,
+      "eval_samples_per_second": 22.682,
+      "eval_steps_per_second": 0.953,
+      "step": 410
+    },
+    {
+      "epoch": 0.5115121344119478,
+      "grad_norm": 3.3563625812530518,
+      "learning_rate": 4.642527675276753e-06,
+      "loss": 1.6751,
+      "step": 411
+    },
+    {
+      "epoch": 0.5127566894835096,
+      "grad_norm": 2.968120574951172,
+      "learning_rate": 4.640221402214023e-06,
+      "loss": 1.663,
+      "step": 412
+    },
+    {
+      "epoch": 0.5140012445550716,
+      "grad_norm": 3.310248374938965,
+      "learning_rate": 4.637915129151292e-06,
+      "loss": 1.658,
+      "step": 413
+    },
+    {
+      "epoch": 0.5152457996266335,
+      "grad_norm": 2.7011170387268066,
+      "learning_rate": 4.635608856088562e-06,
+      "loss": 1.6894,
+      "step": 414
+    },
+    {
+      "epoch": 0.5164903546981954,
+      "grad_norm": 2.804901361465454,
+      "learning_rate": 4.63330258302583e-06,
+      "loss": 1.602,
+      "step": 415
+    },
+    {
+      "epoch": 0.5177349097697573,
+      "grad_norm": 3.2412750720977783,
+      "learning_rate": 4.6309963099631e-06,
+      "loss": 1.6158,
+      "step": 416
+    },
+    {
+      "epoch": 0.5189794648413192,
+      "grad_norm": 2.699909210205078,
+      "learning_rate": 4.62869003690037e-06,
+      "loss": 1.5842,
+      "step": 417
+    },
+    {
+      "epoch": 0.5202240199128811,
+      "grad_norm": 3.065929412841797,
+      "learning_rate": 4.6263837638376384e-06,
+      "loss": 1.6434,
+      "step": 418
+    },
+    {
+      "epoch": 0.5214685749844431,
+      "grad_norm": 2.895305633544922,
+      "learning_rate": 4.624077490774908e-06,
+      "loss": 1.5969,
+      "step": 419
+    },
+    {
+      "epoch": 0.522713130056005,
+      "grad_norm": 3.1590664386749268,
+      "learning_rate": 4.621771217712177e-06,
+      "loss": 1.6211,
+      "step": 420
+    },
+    {
+      "epoch": 0.522713130056005,
+      "eval_loss": 1.7674189805984497,
+      "eval_runtime": 53.328,
+      "eval_samples_per_second": 18.752,
+      "eval_steps_per_second": 0.788,
+      "step": 420
+    },
+    {
+      "epoch": 0.5239576851275669,
+      "grad_norm": 2.7126312255859375,
+      "learning_rate": 4.619464944649447e-06,
+      "loss": 1.6246,
+      "step": 421
+    },
+    {
+      "epoch": 0.5252022401991289,
+      "grad_norm": 2.918508768081665,
+      "learning_rate": 4.617158671586716e-06,
+      "loss": 1.6937,
+      "step": 422
+    },
+    {
+      "epoch": 0.5264467952706907,
+      "grad_norm": 3.008610725402832,
+      "learning_rate": 4.6148523985239856e-06,
+      "loss": 1.6864,
+      "step": 423
+    },
+    {
+      "epoch": 0.5276913503422527,
+      "grad_norm": 2.6612043380737305,
+      "learning_rate": 4.612546125461255e-06,
+      "loss": 1.5785,
+      "step": 424
+    },
+    {
+      "epoch": 0.5289359054138145,
+      "grad_norm": 2.850679874420166,
+      "learning_rate": 4.610239852398524e-06,
+      "loss": 1.5899,
+      "step": 425
+    },
+    {
+      "epoch": 0.5301804604853765,
+      "grad_norm": 2.9205031394958496,
+      "learning_rate": 4.607933579335794e-06,
+      "loss": 1.5867,
+      "step": 426
+    },
+    {
+      "epoch": 0.5314250155569384,
+      "grad_norm": 2.929532527923584,
+      "learning_rate": 4.605627306273063e-06,
+      "loss": 1.6182,
+      "step": 427
+    },
+    {
+      "epoch": 0.5326695706285003,
+      "grad_norm": 2.874936103820801,
+      "learning_rate": 4.603321033210333e-06,
+      "loss": 1.5789,
+      "step": 428
+    },
+    {
+      "epoch": 0.5339141257000622,
+      "grad_norm": 2.8703525066375732,
+      "learning_rate": 4.601014760147602e-06,
+      "loss": 1.5859,
+      "step": 429
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "grad_norm": 2.7076902389526367,
+      "learning_rate": 4.5987084870848715e-06,
+      "loss": 1.644,
+      "step": 430
+    },
+    {
+      "epoch": 0.5351586807716241,
+      "eval_loss": 1.7592095136642456,
+      "eval_runtime": 47.9438,
+      "eval_samples_per_second": 20.858,
+      "eval_steps_per_second": 0.876,
+      "step": 430
+    },
+    {
+      "epoch": 0.536403235843186,
+      "grad_norm": 2.839160680770874,
+      "learning_rate": 4.59640221402214e-06,
+      "loss": 1.6357,
+      "step": 431
+    },
+    {
+      "epoch": 0.537647790914748,
+      "grad_norm": 2.674025535583496,
+      "learning_rate": 4.5940959409594095e-06,
+      "loss": 1.676,
+      "step": 432
+    },
+    {
+      "epoch": 0.5388923459863099,
+      "grad_norm": 3.0619537830352783,
+      "learning_rate": 4.59178966789668e-06,
+      "loss": 1.5617,
+      "step": 433
+    },
+    {
+      "epoch": 0.5401369010578718,
+      "grad_norm": 3.058418035507202,
+      "learning_rate": 4.589483394833948e-06,
+      "loss": 1.5951,
+      "step": 434
+    },
+    {
+      "epoch": 0.5413814561294338,
+      "grad_norm": 3.0046396255493164,
+      "learning_rate": 4.587177121771218e-06,
+      "loss": 1.6473,
+      "step": 435
+    },
+    {
+      "epoch": 0.5426260112009956,
+      "grad_norm": 2.7736752033233643,
+      "learning_rate": 4.584870848708487e-06,
+      "loss": 1.6262,
+      "step": 436
+    },
+    {
+      "epoch": 0.5438705662725576,
+      "grad_norm": 2.7804994583129883,
+      "learning_rate": 4.5825645756457575e-06,
+      "loss": 1.6886,
+      "step": 437
+    },
+    {
+      "epoch": 0.5451151213441194,
+      "grad_norm": 3.0717954635620117,
+      "learning_rate": 4.580258302583026e-06,
+      "loss": 1.6287,
+      "step": 438
+    },
+    {
+      "epoch": 0.5463596764156814,
+      "grad_norm": 2.800243854522705,
+      "learning_rate": 4.5779520295202955e-06,
+      "loss": 1.6169,
+      "step": 439
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "grad_norm": 2.9581644535064697,
+      "learning_rate": 4.575645756457565e-06,
+      "loss": 1.6617,
+      "step": 440
+    },
+    {
+      "epoch": 0.5476042314872434,
+      "eval_loss": 1.7642868757247925,
+      "eval_runtime": 51.2646,
+      "eval_samples_per_second": 19.507,
+      "eval_steps_per_second": 0.819,
+      "step": 440
+    },
+    {
+      "epoch": 0.5488487865588052,
+      "grad_norm": 3.0067460536956787,
+      "learning_rate": 4.573339483394834e-06,
+      "loss": 1.6482,
+      "step": 441
+    },
+    {
+      "epoch": 0.5500933416303672,
+      "grad_norm": 2.9075019359588623,
+      "learning_rate": 4.571033210332104e-06,
+      "loss": 1.6595,
+      "step": 442
+    },
+    {
+      "epoch": 0.551337896701929,
+      "grad_norm": 2.7788755893707275,
+      "learning_rate": 4.568726937269373e-06,
+      "loss": 1.5733,
+      "step": 443
+    },
+    {
+      "epoch": 0.552582451773491,
+      "grad_norm": 2.7775425910949707,
+      "learning_rate": 4.566420664206643e-06,
+      "loss": 1.6925,
+      "step": 444
+    },
+    {
+      "epoch": 0.5538270068450529,
+      "grad_norm": 2.6949567794799805,
+      "learning_rate": 4.564114391143912e-06,
+      "loss": 1.5738,
+      "step": 445
+    },
+    {
+      "epoch": 0.5550715619166148,
+      "grad_norm": 2.7093007564544678,
+      "learning_rate": 4.5618081180811814e-06,
+      "loss": 1.5848,
+      "step": 446
+    },
+    {
+      "epoch": 0.5563161169881767,
+      "grad_norm": 2.6920173168182373,
+      "learning_rate": 4.559501845018451e-06,
+      "loss": 1.6092,
+      "step": 447
+    },
+    {
+      "epoch": 0.5575606720597387,
+      "grad_norm": 2.733731985092163,
+      "learning_rate": 4.5571955719557194e-06,
+      "loss": 1.6056,
+      "step": 448
+    },
+    {
+      "epoch": 0.5588052271313005,
+      "grad_norm": 2.9156274795532227,
+      "learning_rate": 4.55488929889299e-06,
+      "loss": 1.6436,
+      "step": 449
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "grad_norm": 2.8802616596221924,
+      "learning_rate": 4.552583025830259e-06,
+      "loss": 1.6309,
+      "step": 450
+    },
+    {
+      "epoch": 0.5600497822028625,
+      "eval_loss": 1.7486340999603271,
+      "eval_runtime": 47.2219,
+      "eval_samples_per_second": 21.177,
+      "eval_steps_per_second": 0.889,
+      "step": 450
+    },
+    {
+      "epoch": 0.5612943372744243,
+      "grad_norm": 2.729743242263794,
+      "learning_rate": 4.550276752767528e-06,
+      "loss": 1.7171,
+      "step": 451
+    },
+    {
+      "epoch": 0.5625388923459863,
+      "grad_norm": 2.800049066543579,
+      "learning_rate": 4.547970479704797e-06,
+      "loss": 1.6098,
+      "step": 452
+    },
+    {
+      "epoch": 0.5637834474175483,
+      "grad_norm": 2.9659311771392822,
+      "learning_rate": 4.5456642066420666e-06,
+      "loss": 1.6646,
+      "step": 453
+    },
+    {
+      "epoch": 0.5650280024891101,
+      "grad_norm": 2.5417754650115967,
+      "learning_rate": 4.543357933579336e-06,
+      "loss": 1.5847,
+      "step": 454
+    },
+    {
+      "epoch": 0.5662725575606721,
+      "grad_norm": 2.8471338748931885,
+      "learning_rate": 4.541051660516605e-06,
+      "loss": 1.6235,
+      "step": 455
+    },
+    {
+      "epoch": 0.5675171126322339,
+      "grad_norm": 2.7039637565612793,
+      "learning_rate": 4.538745387453875e-06,
+      "loss": 1.5613,
+      "step": 456
+    },
+    {
+      "epoch": 0.5687616677037959,
+      "grad_norm": 3.0383341312408447,
+      "learning_rate": 4.536439114391144e-06,
+      "loss": 1.6485,
+      "step": 457
+    },
+    {
+      "epoch": 0.5700062227753578,
+      "grad_norm": 2.528388261795044,
+      "learning_rate": 4.534132841328414e-06,
+      "loss": 1.6117,
+      "step": 458
+    },
+    {
+      "epoch": 0.5712507778469197,
+      "grad_norm": 2.7623119354248047,
+      "learning_rate": 4.531826568265683e-06,
+      "loss": 1.6017,
+      "step": 459
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "grad_norm": 2.9213945865631104,
+      "learning_rate": 4.5295202952029525e-06,
+      "loss": 1.643,
+      "step": 460
+    },
+    {
+      "epoch": 0.5724953329184816,
+      "eval_loss": 1.747575044631958,
+      "eval_runtime": 45.2844,
+      "eval_samples_per_second": 22.083,
+      "eval_steps_per_second": 0.927,
+      "step": 460
+    },
+    {
+      "epoch": 0.5737398879900436,
+      "grad_norm": 2.664275884628296,
+      "learning_rate": 4.527214022140222e-06,
+      "loss": 1.6154,
+      "step": 461
+    },
+    {
+      "epoch": 0.5749844430616055,
+      "grad_norm": 2.5891788005828857,
+      "learning_rate": 4.524907749077491e-06,
+      "loss": 1.5975,
+      "step": 462
+    },
+    {
+      "epoch": 0.5762289981331674,
+      "grad_norm": 2.8126487731933594,
+      "learning_rate": 4.522601476014761e-06,
+      "loss": 1.6306,
+      "step": 463
+    },
+    {
+      "epoch": 0.5774735532047293,
+      "grad_norm": 2.799391031265259,
+      "learning_rate": 4.520295202952029e-06,
+      "loss": 1.6139,
+      "step": 464
+    },
+    {
+      "epoch": 0.5787181082762912,
+      "grad_norm": 2.7323718070983887,
+      "learning_rate": 4.5179889298893e-06,
+      "loss": 1.632,
+      "step": 465
+    },
+    {
+      "epoch": 0.5799626633478532,
+      "grad_norm": 2.601386785507202,
+      "learning_rate": 4.515682656826569e-06,
+      "loss": 1.6139,
+      "step": 466
+    },
+    {
+      "epoch": 0.581207218419415,
+      "grad_norm": 2.562162399291992,
+      "learning_rate": 4.513376383763838e-06,
+      "loss": 1.5168,
+      "step": 467
+    },
+    {
+      "epoch": 0.582451773490977,
+      "grad_norm": 2.6912002563476562,
+      "learning_rate": 4.511070110701107e-06,
+      "loss": 1.6249,
+      "step": 468
+    },
+    {
+      "epoch": 0.583696328562539,
+      "grad_norm": 2.6971495151519775,
+      "learning_rate": 4.5087638376383765e-06,
+      "loss": 1.616,
+      "step": 469
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "grad_norm": 2.6898910999298096,
+      "learning_rate": 4.506457564575646e-06,
+      "loss": 1.614,
+      "step": 470
+    },
+    {
+      "epoch": 0.5849408836341008,
+      "eval_loss": 1.7460769414901733,
+      "eval_runtime": 45.3055,
+      "eval_samples_per_second": 22.072,
+      "eval_steps_per_second": 0.927,
+      "step": 470
+    },
+    {
+      "epoch": 0.5861854387056628,
+      "grad_norm": 2.698180675506592,
+      "learning_rate": 4.504151291512915e-06,
+      "loss": 1.5993,
+      "step": 471
+    },
+    {
+      "epoch": 0.5874299937772246,
+      "grad_norm": 2.7125210762023926,
+      "learning_rate": 4.501845018450185e-06,
+      "loss": 1.6113,
+      "step": 472
+    },
+    {
+      "epoch": 0.5886745488487866,
+      "grad_norm": 2.754445791244507,
+      "learning_rate": 4.499538745387454e-06,
+      "loss": 1.6719,
+      "step": 473
+    },
+    {
+      "epoch": 0.5899191039203485,
+      "grad_norm": 2.6979362964630127,
+      "learning_rate": 4.497232472324724e-06,
+      "loss": 1.608,
+      "step": 474
+    },
+    {
+      "epoch": 0.5911636589919104,
+      "grad_norm": 2.7694528102874756,
+      "learning_rate": 4.494926199261993e-06,
+      "loss": 1.5695,
+      "step": 475
+    },
+    {
+      "epoch": 0.5924082140634723,
+      "grad_norm": 2.653353214263916,
+      "learning_rate": 4.4926199261992624e-06,
+      "loss": 1.5523,
+      "step": 476
+    },
+    {
+      "epoch": 0.5936527691350342,
+      "grad_norm": 2.6632070541381836,
+      "learning_rate": 4.490313653136532e-06,
+      "loss": 1.5675,
+      "step": 477
+    },
+    {
+      "epoch": 0.5948973242065961,
+      "grad_norm": 3.038543939590454,
+      "learning_rate": 4.488007380073801e-06,
+      "loss": 1.5692,
+      "step": 478
+    },
+    {
+      "epoch": 0.5961418792781581,
+      "grad_norm": 2.8123953342437744,
+      "learning_rate": 4.485701107011071e-06,
+      "loss": 1.6223,
+      "step": 479
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "grad_norm": 2.5845773220062256,
+      "learning_rate": 4.483394833948339e-06,
+      "loss": 1.5773,
+      "step": 480
+    },
+    {
+      "epoch": 0.5973864343497199,
+      "eval_loss": 1.7419319152832031,
+      "eval_runtime": 44.6842,
+      "eval_samples_per_second": 22.379,
+      "eval_steps_per_second": 0.94,
+      "step": 480
+    },
+    {
+      "epoch": 0.5986309894212819,
+      "grad_norm": 3.069936752319336,
+      "learning_rate": 4.4810885608856096e-06,
+      "loss": 1.6323,
+      "step": 481
+    },
+    {
+      "epoch": 0.5998755444928439,
+      "grad_norm": 2.9085781574249268,
+      "learning_rate": 4.478782287822879e-06,
+      "loss": 1.5845,
+      "step": 482
+    },
+    {
+      "epoch": 0.6011200995644057,
+      "grad_norm": 2.8568010330200195,
+      "learning_rate": 4.4764760147601476e-06,
+      "loss": 1.5898,
+      "step": 483
+    },
+    {
+      "epoch": 0.6023646546359677,
+      "grad_norm": 3.089081048965454,
+      "learning_rate": 4.474169741697417e-06,
+      "loss": 1.6622,
+      "step": 484
+    },
+    {
+      "epoch": 0.6036092097075295,
+      "grad_norm": 2.8515470027923584,
+      "learning_rate": 4.471863468634686e-06,
+      "loss": 1.5859,
+      "step": 485
+    },
+    {
+      "epoch": 0.6048537647790915,
+      "grad_norm": 2.647765874862671,
+      "learning_rate": 4.469557195571957e-06,
+      "loss": 1.5926,
+      "step": 486
+    },
+    {
+      "epoch": 0.6060983198506534,
+      "grad_norm": 2.61676025390625,
+      "learning_rate": 4.467250922509225e-06,
+      "loss": 1.5684,
+      "step": 487
+    },
+    {
+      "epoch": 0.6073428749222153,
+      "grad_norm": 2.7127039432525635,
+      "learning_rate": 4.464944649446495e-06,
+      "loss": 1.5702,
+      "step": 488
+    },
+    {
+      "epoch": 0.6085874299937772,
+      "grad_norm": 2.9932055473327637,
+      "learning_rate": 4.462638376383764e-06,
+      "loss": 1.5976,
+      "step": 489
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "grad_norm": 2.6508774757385254,
+      "learning_rate": 4.4603321033210335e-06,
+      "loss": 1.586,
+      "step": 490
+    },
+    {
+      "epoch": 0.6098319850653391,
+      "eval_loss": 1.7357326745986938,
+      "eval_runtime": 45.393,
+      "eval_samples_per_second": 22.03,
+      "eval_steps_per_second": 0.925,
+      "step": 490
+    },
+    {
+      "epoch": 0.611076540136901,
+      "grad_norm": 2.7626044750213623,
+      "learning_rate": 4.458025830258303e-06,
+      "loss": 1.5773,
+      "step": 491
+    },
+    {
+      "epoch": 0.612321095208463,
+      "grad_norm": 2.595003604888916,
+      "learning_rate": 4.455719557195572e-06,
+      "loss": 1.601,
+      "step": 492
+    },
+    {
+      "epoch": 0.6135656502800249,
+      "grad_norm": 2.595767021179199,
+      "learning_rate": 4.453413284132842e-06,
+      "loss": 1.6287,
+      "step": 493
+    },
+    {
+      "epoch": 0.6148102053515868,
+      "grad_norm": 2.755845308303833,
+      "learning_rate": 4.451107011070111e-06,
+      "loss": 1.593,
+      "step": 494
+    },
+    {
+      "epoch": 0.6160547604231488,
+      "grad_norm": 2.7302653789520264,
+      "learning_rate": 4.448800738007381e-06,
+      "loss": 1.5768,
+      "step": 495
+    },
+    {
+      "epoch": 0.6172993154947106,
+      "grad_norm": 2.6493024826049805,
+      "learning_rate": 4.446494464944649e-06,
+      "loss": 1.6059,
+      "step": 496
+    },
+    {
+      "epoch": 0.6185438705662726,
+      "grad_norm": 2.8633735179901123,
+      "learning_rate": 4.4441881918819195e-06,
+      "loss": 1.598,
+      "step": 497
+    },
+    {
+      "epoch": 0.6197884256378344,
+      "grad_norm": 2.65639066696167,
+      "learning_rate": 4.441881918819189e-06,
+      "loss": 1.6036,
+      "step": 498
+    },
+    {
+      "epoch": 0.6210329807093964,
+      "grad_norm": 2.6867759227752686,
+      "learning_rate": 4.439575645756458e-06,
+      "loss": 1.5676,
+      "step": 499
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "grad_norm": 2.7986443042755127,
+      "learning_rate": 4.437269372693727e-06,
+      "loss": 1.524,
+      "step": 500
+    },
+    {
+      "epoch": 0.6222775357809583,
+      "eval_loss": 1.7449125051498413,
+      "eval_runtime": 43.9329,
+      "eval_samples_per_second": 22.762,
+      "eval_steps_per_second": 0.956,
+      "step": 500
+    },
+    {
+      "epoch": 0.6235220908525202,
+      "grad_norm": 2.665905714035034,
+      "learning_rate": 4.434963099630996e-06,
+      "loss": 1.5635,
+      "step": 501
+    },
+    {
+      "epoch": 0.6247666459240822,
+      "grad_norm": 2.902435541152954,
+      "learning_rate": 4.432656826568267e-06,
+      "loss": 1.6377,
+      "step": 502
+    },
+    {
+      "epoch": 0.6260112009956441,
+      "grad_norm": 2.6872262954711914,
+      "learning_rate": 4.430350553505535e-06,
+      "loss": 1.5553,
+      "step": 503
+    },
+    {
+      "epoch": 0.627255756067206,
+      "grad_norm": 2.676621913909912,
+      "learning_rate": 4.428044280442805e-06,
+      "loss": 1.6101,
+      "step": 504
+    },
+    {
+      "epoch": 0.6285003111387679,
+      "grad_norm": 2.756347179412842,
+      "learning_rate": 4.425738007380074e-06,
+      "loss": 1.5884,
+      "step": 505
+    },
+    {
+      "epoch": 0.6297448662103298,
+      "grad_norm": 2.856882333755493,
+      "learning_rate": 4.4234317343173434e-06,
+      "loss": 1.5635,
+      "step": 506
+    },
+    {
+      "epoch": 0.6309894212818917,
+      "grad_norm": 2.7708330154418945,
+      "learning_rate": 4.421125461254613e-06,
+      "loss": 1.6028,
+      "step": 507
+    },
+    {
+      "epoch": 0.6322339763534537,
+      "grad_norm": 2.8167600631713867,
+      "learning_rate": 4.418819188191882e-06,
+      "loss": 1.6026,
+      "step": 508
+    },
+    {
+      "epoch": 0.6334785314250155,
+      "grad_norm": 2.54194974899292,
+      "learning_rate": 4.416512915129152e-06,
+      "loss": 1.6001,
+      "step": 509
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "grad_norm": 2.683037519454956,
+      "learning_rate": 4.414206642066421e-06,
+      "loss": 1.6084,
+      "step": 510
+    },
+    {
+      "epoch": 0.6347230864965775,
+      "eval_loss": 1.7376186847686768,
+      "eval_runtime": 42.3026,
+      "eval_samples_per_second": 23.639,
+      "eval_steps_per_second": 0.993,
+      "step": 510
+    },
+    {
+      "epoch": 0.6359676415681393,
+      "grad_norm": 2.82627010345459,
+      "learning_rate": 4.4119003690036905e-06,
+      "loss": 1.6258,
+      "step": 511
+    },
+    {
+      "epoch": 0.6372121966397013,
+      "grad_norm": 2.6139848232269287,
+      "learning_rate": 4.40959409594096e-06,
+      "loss": 1.6029,
+      "step": 512
+    },
+    {
+      "epoch": 0.6384567517112633,
+      "grad_norm": 2.646712303161621,
+      "learning_rate": 4.407287822878229e-06,
+      "loss": 1.5602,
+      "step": 513
+    },
+    {
+      "epoch": 0.6397013067828251,
+      "grad_norm": 2.910935163497925,
+      "learning_rate": 4.404981549815499e-06,
+      "loss": 1.5757,
+      "step": 514
+    },
+    {
+      "epoch": 0.6409458618543871,
+      "grad_norm": 2.6001148223876953,
+      "learning_rate": 4.402675276752768e-06,
+      "loss": 1.5857,
+      "step": 515
+    },
+    {
+      "epoch": 0.642190416925949,
+      "grad_norm": 2.8525235652923584,
+      "learning_rate": 4.400369003690037e-06,
+      "loss": 1.6092,
+      "step": 516
+    },
+    {
+      "epoch": 0.6434349719975109,
+      "grad_norm": 2.5335488319396973,
+      "learning_rate": 4.398062730627306e-06,
+      "loss": 1.5849,
+      "step": 517
+    },
+    {
+      "epoch": 0.6446795270690728,
+      "grad_norm": 2.5787103176116943,
+      "learning_rate": 4.3957564575645765e-06,
+      "loss": 1.634,
+      "step": 518
+    },
+    {
+      "epoch": 0.6459240821406347,
+      "grad_norm": 2.6188197135925293,
+      "learning_rate": 4.393450184501845e-06,
+      "loss": 1.5477,
+      "step": 519
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "grad_norm": 2.6548666954040527,
+      "learning_rate": 4.3911439114391145e-06,
+      "loss": 1.6133,
+      "step": 520
+    },
+    {
+      "epoch": 0.6471686372121966,
+      "eval_loss": 1.7393039464950562,
+      "eval_runtime": 45.9234,
+      "eval_samples_per_second": 21.775,
+      "eval_steps_per_second": 0.915,
+      "step": 520
+    },
+    {
+      "epoch": 0.6484131922837586,
+      "grad_norm": 2.907928228378296,
+      "learning_rate": 4.388837638376384e-06,
+      "loss": 1.587,
+      "step": 521
+    },
+    {
+      "epoch": 0.6496577473553204,
+      "grad_norm": 2.7899692058563232,
+      "learning_rate": 4.386531365313653e-06,
+      "loss": 1.6126,
+      "step": 522
+    },
+    {
+      "epoch": 0.6509023024268824,
+      "grad_norm": 2.680147886276245,
+      "learning_rate": 4.384225092250923e-06,
+      "loss": 1.565,
+      "step": 523
+    },
+    {
+      "epoch": 0.6521468574984443,
+      "grad_norm": 2.6590754985809326,
+      "learning_rate": 4.381918819188192e-06,
+      "loss": 1.6127,
+      "step": 524
+    },
+    {
+      "epoch": 0.6533914125700062,
+      "grad_norm": 2.6638906002044678,
+      "learning_rate": 4.379612546125462e-06,
+      "loss": 1.5615,
+      "step": 525
+    },
+    {
+      "epoch": 0.6546359676415682,
+      "grad_norm": 2.6518194675445557,
+      "learning_rate": 4.377306273062731e-06,
+      "loss": 1.6202,
+      "step": 526
+    },
+    {
+      "epoch": 0.65588052271313,
+      "grad_norm": 2.5323619842529297,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 1.5305,
+      "step": 527
+    },
+    {
+      "epoch": 0.657125077784692,
+      "grad_norm": 2.6871442794799805,
+      "learning_rate": 4.37269372693727e-06,
+      "loss": 1.6043,
+      "step": 528
+    },
+    {
+      "epoch": 0.6583696328562539,
+      "grad_norm": 2.760746717453003,
+      "learning_rate": 4.370387453874539e-06,
+      "loss": 1.6059,
+      "step": 529
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "grad_norm": 2.780482769012451,
+      "learning_rate": 4.368081180811809e-06,
+      "loss": 1.6082,
+      "step": 530
+    },
+    {
+      "epoch": 0.6596141879278158,
+      "eval_loss": 1.7219713926315308,
+      "eval_runtime": 51.8478,
+      "eval_samples_per_second": 19.287,
+      "eval_steps_per_second": 0.81,
+      "step": 530
+    },
+    {
+      "epoch": 0.6608587429993777,
+      "grad_norm": 2.605890989303589,
+      "learning_rate": 4.365774907749078e-06,
+      "loss": 1.5554,
+      "step": 531
+    },
+    {
+      "epoch": 0.6621032980709396,
+      "grad_norm": 2.731555461883545,
+      "learning_rate": 4.363468634686347e-06,
+      "loss": 1.606,
+      "step": 532
+    },
+    {
+      "epoch": 0.6633478531425016,
+      "grad_norm": 2.8356943130493164,
+      "learning_rate": 4.361162361623616e-06,
+      "loss": 1.5601,
+      "step": 533
+    },
+    {
+      "epoch": 0.6645924082140635,
+      "grad_norm": 2.7196593284606934,
+      "learning_rate": 4.3588560885608864e-06,
+      "loss": 1.5722,
+      "step": 534
+    },
+    {
+      "epoch": 0.6658369632856254,
+      "grad_norm": 2.621371269226074,
+      "learning_rate": 4.356549815498156e-06,
+      "loss": 1.6194,
+      "step": 535
+    },
+    {
+      "epoch": 0.6670815183571873,
+      "grad_norm": 2.653916120529175,
+      "learning_rate": 4.354243542435424e-06,
+      "loss": 1.5592,
+      "step": 536
+    },
+    {
+      "epoch": 0.6683260734287492,
+      "grad_norm": 2.88431453704834,
+      "learning_rate": 4.351937269372694e-06,
+      "loss": 1.6296,
+      "step": 537
+    },
+    {
+      "epoch": 0.6695706285003111,
+      "grad_norm": 2.667130470275879,
+      "learning_rate": 4.349630996309963e-06,
+      "loss": 1.5624,
+      "step": 538
+    },
+    {
+      "epoch": 0.6708151835718731,
+      "grad_norm": 2.6453566551208496,
+      "learning_rate": 4.347324723247233e-06,
+      "loss": 1.6025,
+      "step": 539
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "grad_norm": 3.032271146774292,
+      "learning_rate": 4.345018450184502e-06,
+      "loss": 1.6771,
+      "step": 540
+    },
+    {
+      "epoch": 0.6720597386434349,
+      "eval_loss": 1.721895456314087,
+      "eval_runtime": 50.62,
+      "eval_samples_per_second": 19.755,
+      "eval_steps_per_second": 0.83,
+      "step": 540
+    },
+    {
+      "epoch": 0.6733042937149969,
+      "grad_norm": 2.8476362228393555,
+      "learning_rate": 4.3427121771217715e-06,
+      "loss": 1.5663,
+      "step": 541
+    },
+    {
+      "epoch": 0.6745488487865589,
+      "grad_norm": 2.615602970123291,
+      "learning_rate": 4.340405904059041e-06,
+      "loss": 1.5815,
+      "step": 542
+    },
+    {
+      "epoch": 0.6757934038581207,
+      "grad_norm": 3.221571683883667,
+      "learning_rate": 4.33809963099631e-06,
+      "loss": 1.5955,
+      "step": 543
+    },
+    {
+      "epoch": 0.6770379589296827,
+      "grad_norm": 2.602994203567505,
+      "learning_rate": 4.33579335793358e-06,
+      "loss": 1.5667,
+      "step": 544
+    },
+    {
+      "epoch": 0.6782825140012445,
+      "grad_norm": 2.9380433559417725,
+      "learning_rate": 4.333487084870848e-06,
+      "loss": 1.5651,
+      "step": 545
+    },
+    {
+      "epoch": 0.6795270690728065,
+      "grad_norm": 3.0689425468444824,
+      "learning_rate": 4.331180811808119e-06,
+      "loss": 1.6003,
+      "step": 546
+    },
+    {
+      "epoch": 0.6807716241443684,
+      "grad_norm": 2.770317316055298,
+      "learning_rate": 4.328874538745388e-06,
+      "loss": 1.5576,
+      "step": 547
+    },
+    {
+      "epoch": 0.6820161792159303,
+      "grad_norm": 2.8631815910339355,
+      "learning_rate": 4.3265682656826575e-06,
+      "loss": 1.5308,
+      "step": 548
+    },
+    {
+      "epoch": 0.6832607342874922,
+      "grad_norm": 2.785576820373535,
+      "learning_rate": 4.324261992619926e-06,
+      "loss": 1.6113,
+      "step": 549
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "grad_norm": 2.723919630050659,
+      "learning_rate": 4.321955719557196e-06,
+      "loss": 1.5606,
+      "step": 550
+    },
+    {
+      "epoch": 0.6845052893590542,
+      "eval_loss": 1.721803069114685,
+      "eval_runtime": 51.6036,
+      "eval_samples_per_second": 19.378,
+      "eval_steps_per_second": 0.814,
+      "step": 550
+    },
+    {
+      "epoch": 0.685749844430616,
+      "grad_norm": 3.081198215484619,
+      "learning_rate": 4.319649446494466e-06,
+      "loss": 1.5643,
+      "step": 551
+    },
+    {
+      "epoch": 0.686994399502178,
+      "grad_norm": 2.757392168045044,
+      "learning_rate": 4.317343173431734e-06,
+      "loss": 1.5909,
+      "step": 552
+    },
+    {
+      "epoch": 0.6882389545737398,
+      "grad_norm": 3.038753032684326,
+      "learning_rate": 4.315036900369004e-06,
+      "loss": 1.6023,
+      "step": 553
+    },
+    {
+      "epoch": 0.6894835096453018,
+      "grad_norm": 2.8246452808380127,
+      "learning_rate": 4.312730627306273e-06,
+      "loss": 1.5548,
+      "step": 554
+    },
+    {
+      "epoch": 0.6907280647168638,
+      "grad_norm": 2.589320182800293,
+      "learning_rate": 4.310424354243543e-06,
+      "loss": 1.5424,
+      "step": 555
+    },
+    {
+      "epoch": 0.6919726197884256,
+      "grad_norm": 2.6413373947143555,
+      "learning_rate": 4.308118081180812e-06,
+      "loss": 1.6243,
+      "step": 556
+    },
+    {
+      "epoch": 0.6932171748599876,
+      "grad_norm": 2.764784336090088,
+      "learning_rate": 4.3058118081180815e-06,
+      "loss": 1.589,
+      "step": 557
+    },
+    {
+      "epoch": 0.6944617299315494,
+      "grad_norm": 2.806821823120117,
+      "learning_rate": 4.303505535055351e-06,
+      "loss": 1.621,
+      "step": 558
+    },
+    {
+      "epoch": 0.6957062850031114,
+      "grad_norm": 2.718017578125,
+      "learning_rate": 4.30119926199262e-06,
+      "loss": 1.5773,
+      "step": 559
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "grad_norm": 2.7620160579681396,
+      "learning_rate": 4.29889298892989e-06,
+      "loss": 1.5868,
+      "step": 560
+    },
+    {
+      "epoch": 0.6969508400746733,
+      "eval_loss": 1.7152249813079834,
+      "eval_runtime": 52.0463,
+      "eval_samples_per_second": 19.214,
+      "eval_steps_per_second": 0.807,
+      "step": 560
+    },
+    {
+      "epoch": 0.6981953951462352,
+      "grad_norm": 2.66686749458313,
+      "learning_rate": 4.296586715867159e-06,
+      "loss": 1.534,
+      "step": 561
+    },
+    {
+      "epoch": 0.6994399502177971,
+      "grad_norm": 2.812577247619629,
+      "learning_rate": 4.2942804428044286e-06,
+      "loss": 1.5501,
+      "step": 562
+    },
+    {
+      "epoch": 0.7006845052893591,
+      "grad_norm": 2.578508138656616,
+      "learning_rate": 4.291974169741698e-06,
+      "loss": 1.6286,
+      "step": 563
+    },
+    {
+      "epoch": 0.701929060360921,
+      "grad_norm": 2.524249315261841,
+      "learning_rate": 4.289667896678967e-06,
+      "loss": 1.5719,
+      "step": 564
+    },
+    {
+      "epoch": 0.7031736154324829,
+      "grad_norm": 2.827235460281372,
+      "learning_rate": 4.287361623616236e-06,
+      "loss": 1.5333,
+      "step": 565
+    },
+    {
+      "epoch": 0.7044181705040448,
+      "grad_norm": 2.6359963417053223,
+      "learning_rate": 4.285055350553506e-06,
+      "loss": 1.5849,
+      "step": 566
+    },
+    {
+      "epoch": 0.7056627255756067,
+      "grad_norm": 2.930530071258545,
+      "learning_rate": 4.282749077490776e-06,
+      "loss": 1.5672,
+      "step": 567
+    },
+    {
+      "epoch": 0.7069072806471687,
+      "grad_norm": 2.750102996826172,
+      "learning_rate": 4.280442804428044e-06,
+      "loss": 1.5696,
+      "step": 568
+    },
+    {
+      "epoch": 0.7081518357187305,
+      "grad_norm": 2.869690418243408,
+      "learning_rate": 4.278136531365314e-06,
+      "loss": 1.5689,
+      "step": 569
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "grad_norm": 2.954852819442749,
+      "learning_rate": 4.275830258302583e-06,
+      "loss": 1.5931,
+      "step": 570
+    },
+    {
+      "epoch": 0.7093963907902925,
+      "eval_loss": 1.7143300771713257,
+      "eval_runtime": 52.7293,
+      "eval_samples_per_second": 18.965,
+      "eval_steps_per_second": 0.797,
+      "step": 570
+    },
+    {
+      "epoch": 0.7106409458618543,
+      "grad_norm": 2.702223539352417,
+      "learning_rate": 4.273523985239853e-06,
+      "loss": 1.5635,
+      "step": 571
+    },
+    {
+      "epoch": 0.7118855009334163,
+      "grad_norm": 2.689995050430298,
+      "learning_rate": 4.271217712177122e-06,
+      "loss": 1.5545,
+      "step": 572
+    },
+    {
+      "epoch": 0.7131300560049783,
+      "grad_norm": 2.68979811668396,
+      "learning_rate": 4.268911439114391e-06,
+      "loss": 1.5404,
+      "step": 573
+    },
+    {
+      "epoch": 0.7143746110765401,
+      "grad_norm": 2.7477986812591553,
+      "learning_rate": 4.266605166051661e-06,
+      "loss": 1.5719,
+      "step": 574
+    },
+    {
+      "epoch": 0.7156191661481021,
+      "grad_norm": 2.975778341293335,
+      "learning_rate": 4.26429889298893e-06,
+      "loss": 1.5587,
+      "step": 575
+    },
+    {
+      "epoch": 0.716863721219664,
+      "grad_norm": 2.658170700073242,
+      "learning_rate": 4.2619926199262e-06,
+      "loss": 1.5746,
+      "step": 576
+    },
+    {
+      "epoch": 0.7181082762912259,
+      "grad_norm": 2.8825011253356934,
+      "learning_rate": 4.259686346863469e-06,
+      "loss": 1.5246,
+      "step": 577
+    },
+    {
+      "epoch": 0.7193528313627878,
+      "grad_norm": 2.845280170440674,
+      "learning_rate": 4.2573800738007385e-06,
+      "loss": 1.5703,
+      "step": 578
+    },
+    {
+      "epoch": 0.7205973864343497,
+      "grad_norm": 2.660616636276245,
+      "learning_rate": 4.255073800738008e-06,
+      "loss": 1.5373,
+      "step": 579
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "grad_norm": 2.749447822570801,
+      "learning_rate": 4.252767527675277e-06,
+      "loss": 1.59,
+      "step": 580
+    },
+    {
+      "epoch": 0.7218419415059116,
+      "eval_loss": 1.7149444818496704,
+      "eval_runtime": 52.2099,
+      "eval_samples_per_second": 19.153,
+      "eval_steps_per_second": 0.804,
+      "step": 580
+    },
+    {
+      "epoch": 0.7230864965774736,
+      "grad_norm": 2.813328742980957,
+      "learning_rate": 4.250461254612546e-06,
+      "loss": 1.5694,
+      "step": 581
+    },
+    {
+      "epoch": 0.7243310516490354,
+      "grad_norm": 2.823866844177246,
+      "learning_rate": 4.248154981549816e-06,
+      "loss": 1.5143,
+      "step": 582
+    },
+    {
+      "epoch": 0.7255756067205974,
+      "grad_norm": 2.71337890625,
+      "learning_rate": 4.245848708487086e-06,
+      "loss": 1.5915,
+      "step": 583
+    },
+    {
+      "epoch": 0.7268201617921594,
+      "grad_norm": 2.718085765838623,
+      "learning_rate": 4.243542435424355e-06,
+      "loss": 1.604,
+      "step": 584
+    },
+    {
+      "epoch": 0.7280647168637212,
+      "grad_norm": 2.5741796493530273,
+      "learning_rate": 4.241236162361624e-06,
+      "loss": 1.5398,
+      "step": 585
+    },
+    {
+      "epoch": 0.7293092719352832,
+      "grad_norm": 2.727114200592041,
+      "learning_rate": 4.238929889298893e-06,
+      "loss": 1.5886,
+      "step": 586
+    },
+    {
+      "epoch": 0.730553827006845,
+      "grad_norm": 2.6703338623046875,
+      "learning_rate": 4.236623616236163e-06,
+      "loss": 1.5959,
+      "step": 587
+    },
+    {
+      "epoch": 0.731798382078407,
+      "grad_norm": 2.750326633453369,
+      "learning_rate": 4.234317343173432e-06,
+      "loss": 1.5916,
+      "step": 588
+    },
+    {
+      "epoch": 0.7330429371499689,
+      "grad_norm": 2.6680715084075928,
+      "learning_rate": 4.232011070110701e-06,
+      "loss": 1.562,
+      "step": 589
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "grad_norm": 2.6603832244873047,
+      "learning_rate": 4.229704797047971e-06,
+      "loss": 1.6121,
+      "step": 590
+    },
+    {
+      "epoch": 0.7342874922215308,
+      "eval_loss": 1.7144734859466553,
+      "eval_runtime": 50.5317,
+      "eval_samples_per_second": 19.79,
+      "eval_steps_per_second": 0.831,
+      "step": 590
+    },
+    {
+      "epoch": 0.7355320472930927,
+      "grad_norm": 2.7868523597717285,
+      "learning_rate": 4.22739852398524e-06,
+      "loss": 1.5488,
+      "step": 591
+    },
+    {
+      "epoch": 0.7367766023646546,
+      "grad_norm": 2.755270004272461,
+      "learning_rate": 4.2250922509225096e-06,
+      "loss": 1.5801,
+      "step": 592
+    },
+    {
+      "epoch": 0.7380211574362165,
+      "grad_norm": 2.7614126205444336,
+      "learning_rate": 4.222785977859779e-06,
+      "loss": 1.5507,
+      "step": 593
+    },
+    {
+      "epoch": 0.7392657125077785,
+      "grad_norm": 2.8674862384796143,
+      "learning_rate": 4.220479704797048e-06,
+      "loss": 1.5807,
+      "step": 594
+    },
+    {
+      "epoch": 0.7405102675793404,
+      "grad_norm": 2.7366859912872314,
+      "learning_rate": 4.218173431734318e-06,
+      "loss": 1.6594,
+      "step": 595
+    },
+    {
+      "epoch": 0.7417548226509023,
+      "grad_norm": 2.6231045722961426,
+      "learning_rate": 4.215867158671587e-06,
+      "loss": 1.5609,
+      "step": 596
+    },
+    {
+      "epoch": 0.7429993777224643,
+      "grad_norm": 2.7694146633148193,
+      "learning_rate": 4.213560885608857e-06,
+      "loss": 1.5889,
+      "step": 597
+    },
+    {
+      "epoch": 0.7442439327940261,
+      "grad_norm": 2.6275386810302734,
+      "learning_rate": 4.211254612546125e-06,
+      "loss": 1.575,
+      "step": 598
+    },
+    {
+      "epoch": 0.7454884878655881,
+      "grad_norm": 2.9413866996765137,
+      "learning_rate": 4.2089483394833955e-06,
+      "loss": 1.589,
+      "step": 599
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "grad_norm": 2.768606185913086,
+      "learning_rate": 4.206642066420665e-06,
+      "loss": 1.5738,
+      "step": 600
+    },
+    {
+      "epoch": 0.7467330429371499,
+      "eval_loss": 1.7148027420043945,
+      "eval_runtime": 54.7031,
+      "eval_samples_per_second": 18.28,
+      "eval_steps_per_second": 0.768,
+      "step": 600
+    },
+    {
+      "epoch": 0.7479775980087119,
+      "grad_norm": 2.7917706966400146,
+      "learning_rate": 4.2043357933579335e-06,
+      "loss": 1.499,
+      "step": 601
+    },
+    {
+      "epoch": 0.7492221530802738,
+      "grad_norm": 2.5982208251953125,
+      "learning_rate": 4.202029520295203e-06,
+      "loss": 1.5113,
+      "step": 602
+    },
+    {
+      "epoch": 0.7504667081518357,
+      "grad_norm": 2.692032814025879,
+      "learning_rate": 4.199723247232473e-06,
+      "loss": 1.4721,
+      "step": 603
+    },
+    {
+      "epoch": 0.7517112632233977,
+      "grad_norm": 2.975860834121704,
+      "learning_rate": 4.197416974169742e-06,
+      "loss": 1.5147,
+      "step": 604
+    },
+    {
+      "epoch": 0.7529558182949595,
+      "grad_norm": 2.758610248565674,
+      "learning_rate": 4.195110701107011e-06,
+      "loss": 1.5674,
+      "step": 605
+    },
+    {
+      "epoch": 0.7542003733665215,
+      "grad_norm": 2.7620184421539307,
+      "learning_rate": 4.192804428044281e-06,
+      "loss": 1.5669,
+      "step": 606
+    },
+    {
+      "epoch": 0.7554449284380834,
+      "grad_norm": 3.043940305709839,
+      "learning_rate": 4.19049815498155e-06,
+      "loss": 1.587,
+      "step": 607
+    },
+    {
+      "epoch": 0.7566894835096453,
+      "grad_norm": 2.7482151985168457,
+      "learning_rate": 4.1881918819188195e-06,
+      "loss": 1.5451,
+      "step": 608
+    },
+    {
+      "epoch": 0.7579340385812072,
+      "grad_norm": 2.7259294986724854,
+      "learning_rate": 4.185885608856089e-06,
+      "loss": 1.5295,
+      "step": 609
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "grad_norm": 2.6340832710266113,
+      "learning_rate": 4.183579335793358e-06,
+      "loss": 1.5335,
+      "step": 610
+    },
+    {
+      "epoch": 0.7591785936527692,
+      "eval_loss": 1.7038393020629883,
+      "eval_runtime": 44.0792,
+      "eval_samples_per_second": 22.686,
+      "eval_steps_per_second": 0.953,
+      "step": 610
+    },
+    {
+      "epoch": 0.760423148724331,
+      "grad_norm": 2.6144909858703613,
+      "learning_rate": 4.181273062730628e-06,
+      "loss": 1.5907,
+      "step": 611
+    },
+    {
+      "epoch": 0.761667703795893,
+      "grad_norm": 2.7754175662994385,
+      "learning_rate": 4.178966789667897e-06,
+      "loss": 1.5773,
+      "step": 612
+    },
+    {
+      "epoch": 0.7629122588674548,
+      "grad_norm": 2.6313252449035645,
+      "learning_rate": 4.176660516605167e-06,
+      "loss": 1.5675,
+      "step": 613
+    },
+    {
+      "epoch": 0.7641568139390168,
+      "grad_norm": 2.549074649810791,
+      "learning_rate": 4.174354243542435e-06,
+      "loss": 1.595,
+      "step": 614
+    },
+    {
+      "epoch": 0.7654013690105788,
+      "grad_norm": 2.611804246902466,
+      "learning_rate": 4.1720479704797054e-06,
+      "loss": 1.5416,
+      "step": 615
+    },
+    {
+      "epoch": 0.7666459240821406,
+      "grad_norm": 2.6322927474975586,
+      "learning_rate": 4.169741697416975e-06,
+      "loss": 1.604,
+      "step": 616
+    },
+    {
+      "epoch": 0.7678904791537026,
+      "grad_norm": 2.5792219638824463,
+      "learning_rate": 4.1674354243542434e-06,
+      "loss": 1.5976,
+      "step": 617
+    },
+    {
+      "epoch": 0.7691350342252644,
+      "grad_norm": 2.845416307449341,
+      "learning_rate": 4.165129151291513e-06,
+      "loss": 1.5945,
+      "step": 618
+    },
+    {
+      "epoch": 0.7703795892968264,
+      "grad_norm": 2.8647871017456055,
+      "learning_rate": 4.162822878228783e-06,
+      "loss": 1.5637,
+      "step": 619
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "grad_norm": 2.590719699859619,
+      "learning_rate": 4.1605166051660526e-06,
+      "loss": 1.556,
+      "step": 620
+    },
+    {
+      "epoch": 0.7716241443683883,
+      "eval_loss": 1.7115222215652466,
+      "eval_runtime": 50.3905,
+      "eval_samples_per_second": 19.845,
+      "eval_steps_per_second": 0.833,
+      "step": 620
+    },
+    {
+      "epoch": 0.7728686994399502,
+      "grad_norm": 2.626723527908325,
+      "learning_rate": 4.158210332103321e-06,
+      "loss": 1.5486,
+      "step": 621
+    },
+    {
+      "epoch": 0.7741132545115121,
+      "grad_norm": 2.713069438934326,
+      "learning_rate": 4.1559040590405906e-06,
+      "loss": 1.5474,
+      "step": 622
+    },
+    {
+      "epoch": 0.7753578095830741,
+      "grad_norm": 2.685563087463379,
+      "learning_rate": 4.15359778597786e-06,
+      "loss": 1.5529,
+      "step": 623
+    },
+    {
+      "epoch": 0.776602364654636,
+      "grad_norm": 2.690586805343628,
+      "learning_rate": 4.151291512915129e-06,
+      "loss": 1.5293,
+      "step": 624
+    },
+    {
+      "epoch": 0.7778469197261979,
+      "grad_norm": 2.865345001220703,
+      "learning_rate": 4.148985239852399e-06,
+      "loss": 1.5095,
+      "step": 625
+    },
+    {
+      "epoch": 0.7790914747977598,
+      "grad_norm": 2.92393159866333,
+      "learning_rate": 4.146678966789668e-06,
+      "loss": 1.5561,
+      "step": 626
+    },
+    {
+      "epoch": 0.7803360298693217,
+      "grad_norm": 2.554480791091919,
+      "learning_rate": 4.144372693726938e-06,
+      "loss": 1.4876,
+      "step": 627
+    },
+    {
+      "epoch": 0.7815805849408837,
+      "grad_norm": 2.6128833293914795,
+      "learning_rate": 4.142066420664207e-06,
+      "loss": 1.5378,
+      "step": 628
+    },
+    {
+      "epoch": 0.7828251400124455,
+      "grad_norm": 3.0110888481140137,
+      "learning_rate": 4.1397601476014765e-06,
+      "loss": 1.5859,
+      "step": 629
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "grad_norm": 2.8152706623077393,
+      "learning_rate": 4.137453874538745e-06,
+      "loss": 1.5841,
+      "step": 630
+    },
+    {
+      "epoch": 0.7840696950840075,
+      "eval_loss": 1.702181339263916,
+      "eval_runtime": 53.6926,
+      "eval_samples_per_second": 18.625,
+      "eval_steps_per_second": 0.782,
+      "step": 630
+    },
+    {
+      "epoch": 0.7853142501555694,
+      "grad_norm": 2.8634233474731445,
+      "learning_rate": 4.135147601476015e-06,
+      "loss": 1.564,
+      "step": 631
+    },
+    {
+      "epoch": 0.7865588052271313,
+      "grad_norm": 2.8939666748046875,
+      "learning_rate": 4.132841328413285e-06,
+      "loss": 1.5859,
+      "step": 632
+    },
+    {
+      "epoch": 0.7878033602986932,
+      "grad_norm": 2.6560444831848145,
+      "learning_rate": 4.130535055350554e-06,
+      "loss": 1.5199,
+      "step": 633
+    },
+    {
+      "epoch": 0.7890479153702551,
+      "grad_norm": 2.964721202850342,
+      "learning_rate": 4.128228782287823e-06,
+      "loss": 1.5888,
+      "step": 634
+    },
+    {
+      "epoch": 0.790292470441817,
+      "grad_norm": 2.74668288230896,
+      "learning_rate": 4.125922509225092e-06,
+      "loss": 1.5847,
+      "step": 635
+    },
+    {
+      "epoch": 0.791537025513379,
+      "grad_norm": 2.723123550415039,
+      "learning_rate": 4.1236162361623625e-06,
+      "loss": 1.5166,
+      "step": 636
+    },
+    {
+      "epoch": 0.7927815805849409,
+      "grad_norm": 2.7265713214874268,
+      "learning_rate": 4.121309963099631e-06,
+      "loss": 1.5954,
+      "step": 637
+    },
+    {
+      "epoch": 0.7940261356565028,
+      "grad_norm": 2.979126214981079,
+      "learning_rate": 4.1190036900369005e-06,
+      "loss": 1.5583,
+      "step": 638
+    },
+    {
+      "epoch": 0.7952706907280647,
+      "grad_norm": 2.844376802444458,
+      "learning_rate": 4.11669741697417e-06,
+      "loss": 1.5564,
+      "step": 639
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "grad_norm": 2.7583703994750977,
+      "learning_rate": 4.114391143911439e-06,
+      "loss": 1.5094,
+      "step": 640
+    },
+    {
+      "epoch": 0.7965152457996266,
+      "eval_loss": 1.6986565589904785,
+      "eval_runtime": 47.3549,
+      "eval_samples_per_second": 21.117,
+      "eval_steps_per_second": 0.887,
+      "step": 640
+    },
+    {
+      "epoch": 0.7977598008711886,
+      "grad_norm": 2.8942224979400635,
+      "learning_rate": 4.112084870848709e-06,
+      "loss": 1.5504,
+      "step": 641
+    },
+    {
+      "epoch": 0.7990043559427504,
+      "grad_norm": 2.6590495109558105,
+      "learning_rate": 4.109778597785978e-06,
+      "loss": 1.5227,
+      "step": 642
+    },
+    {
+      "epoch": 0.8002489110143124,
+      "grad_norm": 2.5988378524780273,
+      "learning_rate": 4.107472324723248e-06,
+      "loss": 1.5054,
+      "step": 643
+    },
+    {
+      "epoch": 0.8014934660858744,
+      "grad_norm": 2.787335157394409,
+      "learning_rate": 4.105166051660517e-06,
+      "loss": 1.5844,
+      "step": 644
+    },
+    {
+      "epoch": 0.8027380211574362,
+      "grad_norm": 2.8202896118164062,
+      "learning_rate": 4.1028597785977864e-06,
+      "loss": 1.5596,
+      "step": 645
+    },
+    {
+      "epoch": 0.8039825762289982,
+      "grad_norm": 2.65376615524292,
+      "learning_rate": 4.100553505535056e-06,
+      "loss": 1.5893,
+      "step": 646
+    },
+    {
+      "epoch": 0.80522713130056,
+      "grad_norm": 2.5933308601379395,
+      "learning_rate": 4.098247232472325e-06,
+      "loss": 1.5403,
+      "step": 647
+    },
+    {
+      "epoch": 0.806471686372122,
+      "grad_norm": 2.777070999145508,
+      "learning_rate": 4.095940959409595e-06,
+      "loss": 1.5231,
+      "step": 648
+    },
+    {
+      "epoch": 0.8077162414436839,
+      "grad_norm": 2.6427664756774902,
+      "learning_rate": 4.093634686346864e-06,
+      "loss": 1.5937,
+      "step": 649
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "grad_norm": 2.651561737060547,
+      "learning_rate": 4.091328413284133e-06,
+      "loss": 1.4847,
+      "step": 650
+    },
+    {
+      "epoch": 0.8089607965152458,
+      "eval_loss": 1.7089996337890625,
+      "eval_runtime": 47.4346,
+      "eval_samples_per_second": 21.082,
+      "eval_steps_per_second": 0.885,
+      "step": 650
+    },
+    {
+      "epoch": 0.8102053515868077,
+      "grad_norm": 2.844067335128784,
+      "learning_rate": 4.089022140221402e-06,
+      "loss": 1.488,
+      "step": 651
+    },
+    {
+      "epoch": 0.8114499066583696,
+      "grad_norm": 2.603480577468872,
+      "learning_rate": 4.086715867158672e-06,
+      "loss": 1.5248,
+      "step": 652
+    },
+    {
+      "epoch": 0.8126944617299315,
+      "grad_norm": 2.789322853088379,
+      "learning_rate": 4.084409594095941e-06,
+      "loss": 1.5119,
+      "step": 653
+    },
+    {
+      "epoch": 0.8139390168014935,
+      "grad_norm": 2.7183773517608643,
+      "learning_rate": 4.08210332103321e-06,
+      "loss": 1.5123,
+      "step": 654
+    },
+    {
+      "epoch": 0.8151835718730553,
+      "grad_norm": 2.80613374710083,
+      "learning_rate": 4.07979704797048e-06,
+      "loss": 1.5526,
+      "step": 655
+    },
+    {
+      "epoch": 0.8164281269446173,
+      "grad_norm": 2.641035795211792,
+      "learning_rate": 4.077490774907749e-06,
+      "loss": 1.6127,
+      "step": 656
+    },
+    {
+      "epoch": 0.8176726820161793,
+      "grad_norm": 2.773684024810791,
+      "learning_rate": 4.075184501845019e-06,
+      "loss": 1.5404,
+      "step": 657
+    },
+    {
+      "epoch": 0.8189172370877411,
+      "grad_norm": 2.6951382160186768,
+      "learning_rate": 4.072878228782288e-06,
+      "loss": 1.4928,
+      "step": 658
+    },
+    {
+      "epoch": 0.8201617921593031,
+      "grad_norm": 2.931771755218506,
+      "learning_rate": 4.0705719557195575e-06,
+      "loss": 1.5949,
+      "step": 659
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "grad_norm": 2.8786075115203857,
+      "learning_rate": 4.068265682656827e-06,
+      "loss": 1.5582,
+      "step": 660
+    },
+    {
+      "epoch": 0.8214063472308649,
+      "eval_loss": 1.698564052581787,
+      "eval_runtime": 46.6299,
+      "eval_samples_per_second": 21.445,
+      "eval_steps_per_second": 0.901,
+      "step": 660
+    },
+    {
+      "epoch": 0.8226509023024269,
+      "grad_norm": 2.7402451038360596,
+      "learning_rate": 4.065959409594096e-06,
+      "loss": 1.5562,
+      "step": 661
+    },
+    {
+      "epoch": 0.8238954573739888,
+      "grad_norm": 2.7968692779541016,
+      "learning_rate": 4.063653136531366e-06,
+      "loss": 1.556,
+      "step": 662
+    },
+    {
+      "epoch": 0.8251400124455507,
+      "grad_norm": 2.8233132362365723,
+      "learning_rate": 4.061346863468635e-06,
+      "loss": 1.5527,
+      "step": 663
+    },
+    {
+      "epoch": 0.8263845675171126,
+      "grad_norm": 2.713390827178955,
+      "learning_rate": 4.059040590405905e-06,
+      "loss": 1.6012,
+      "step": 664
+    },
+    {
+      "epoch": 0.8276291225886746,
+      "grad_norm": 2.7783989906311035,
+      "learning_rate": 4.056734317343174e-06,
+      "loss": 1.5247,
+      "step": 665
+    },
+    {
+      "epoch": 0.8288736776602365,
+      "grad_norm": 2.808554172515869,
+      "learning_rate": 4.054428044280443e-06,
+      "loss": 1.5412,
+      "step": 666
+    },
+    {
+      "epoch": 0.8301182327317984,
+      "grad_norm": 2.8476831912994385,
+      "learning_rate": 4.052121771217712e-06,
+      "loss": 1.5307,
+      "step": 667
+    },
+    {
+      "epoch": 0.8313627878033603,
+      "grad_norm": 2.8287601470947266,
+      "learning_rate": 4.049815498154982e-06,
+      "loss": 1.6021,
+      "step": 668
+    },
+    {
+      "epoch": 0.8326073428749222,
+      "grad_norm": 2.975510835647583,
+      "learning_rate": 4.047509225092252e-06,
+      "loss": 1.5474,
+      "step": 669
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "grad_norm": 2.723236560821533,
+      "learning_rate": 4.04520295202952e-06,
+      "loss": 1.5242,
+      "step": 670
+    },
+    {
+      "epoch": 0.8338518979464842,
+      "eval_loss": 1.7005239725112915,
+      "eval_runtime": 49.4504,
+      "eval_samples_per_second": 20.222,
+      "eval_steps_per_second": 0.849,
+      "step": 670
+    },
+    {
+      "epoch": 0.835096453018046,
+      "grad_norm": 2.7521748542785645,
+      "learning_rate": 4.04289667896679e-06,
+      "loss": 1.5467,
+      "step": 671
+    },
+    {
+      "epoch": 0.836341008089608,
+      "grad_norm": 2.7770044803619385,
+      "learning_rate": 4.04059040590406e-06,
+      "loss": 1.5407,
+      "step": 672
+    },
+    {
+      "epoch": 0.8375855631611698,
+      "grad_norm": 2.744323492050171,
+      "learning_rate": 4.038284132841329e-06,
+      "loss": 1.5422,
+      "step": 673
+    },
+    {
+      "epoch": 0.8388301182327318,
+      "grad_norm": 2.6699817180633545,
+      "learning_rate": 4.035977859778598e-06,
+      "loss": 1.5508,
+      "step": 674
+    },
+    {
+      "epoch": 0.8400746733042938,
+      "grad_norm": 2.705273151397705,
+      "learning_rate": 4.0336715867158674e-06,
+      "loss": 1.5099,
+      "step": 675
+    },
+    {
+      "epoch": 0.8413192283758556,
+      "grad_norm": 2.9485747814178467,
+      "learning_rate": 4.031365313653137e-06,
+      "loss": 1.5529,
+      "step": 676
+    },
+    {
+      "epoch": 0.8425637834474176,
+      "grad_norm": 3.1750423908233643,
+      "learning_rate": 4.029059040590406e-06,
+      "loss": 1.5894,
+      "step": 677
+    },
+    {
+      "epoch": 0.8438083385189795,
+      "grad_norm": 2.76448655128479,
+      "learning_rate": 4.026752767527676e-06,
+      "loss": 1.5752,
+      "step": 678
+    },
+    {
+      "epoch": 0.8450528935905414,
+      "grad_norm": 2.676708698272705,
+      "learning_rate": 4.024446494464945e-06,
+      "loss": 1.5487,
+      "step": 679
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "grad_norm": 2.6832051277160645,
+      "learning_rate": 4.0221402214022145e-06,
+      "loss": 1.5055,
+      "step": 680
+    },
+    {
+      "epoch": 0.8462974486621033,
+      "eval_loss": 1.6945017576217651,
+      "eval_runtime": 43.9369,
+      "eval_samples_per_second": 22.76,
+      "eval_steps_per_second": 0.956,
+      "step": 680
+    },
+    {
+      "epoch": 0.8475420037336652,
+      "grad_norm": 2.6812336444854736,
+      "learning_rate": 4.019833948339484e-06,
+      "loss": 1.5315,
+      "step": 681
+    },
+    {
+      "epoch": 0.8487865588052271,
+      "grad_norm": 2.9394326210021973,
+      "learning_rate": 4.017527675276753e-06,
+      "loss": 1.5636,
+      "step": 682
+    },
+    {
+      "epoch": 0.8500311138767891,
+      "grad_norm": 2.944952964782715,
+      "learning_rate": 4.015221402214022e-06,
+      "loss": 1.56,
+      "step": 683
+    },
+    {
+      "epoch": 0.8512756689483509,
+      "grad_norm": 2.7475314140319824,
+      "learning_rate": 4.012915129151292e-06,
+      "loss": 1.5442,
+      "step": 684
+    },
+    {
+      "epoch": 0.8525202240199129,
+      "grad_norm": 3.0572879314422607,
+      "learning_rate": 4.010608856088562e-06,
+      "loss": 1.5023,
+      "step": 685
+    },
+    {
+      "epoch": 0.8537647790914747,
+      "grad_norm": 2.75365948677063,
+      "learning_rate": 4.00830258302583e-06,
+      "loss": 1.5414,
+      "step": 686
+    },
+    {
+      "epoch": 0.8550093341630367,
+      "grad_norm": 2.5714449882507324,
+      "learning_rate": 4.0059963099631e-06,
+      "loss": 1.5027,
+      "step": 687
+    },
+    {
+      "epoch": 0.8562538892345987,
+      "grad_norm": 2.755167007446289,
+      "learning_rate": 4.003690036900369e-06,
+      "loss": 1.5788,
+      "step": 688
+    },
+    {
+      "epoch": 0.8574984443061605,
+      "grad_norm": 2.798967123031616,
+      "learning_rate": 4.0013837638376385e-06,
+      "loss": 1.5162,
+      "step": 689
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "grad_norm": 2.803614854812622,
+      "learning_rate": 3.999077490774908e-06,
+      "loss": 1.5079,
+      "step": 690
+    },
+    {
+      "epoch": 0.8587429993777225,
+      "eval_loss": 1.6924962997436523,
+      "eval_runtime": 44.47,
+      "eval_samples_per_second": 22.487,
+      "eval_steps_per_second": 0.944,
+      "step": 690
+    },
+    {
+      "epoch": 0.8599875544492844,
+      "grad_norm": 2.6898066997528076,
+      "learning_rate": 3.996771217712177e-06,
+      "loss": 1.5478,
+      "step": 691
+    },
+    {
+      "epoch": 0.8612321095208463,
+      "grad_norm": 2.6376144886016846,
+      "learning_rate": 3.994464944649447e-06,
+      "loss": 1.5362,
+      "step": 692
+    },
+    {
+      "epoch": 0.8624766645924082,
+      "grad_norm": 2.7240405082702637,
+      "learning_rate": 3.992158671586716e-06,
+      "loss": 1.5502,
+      "step": 693
+    },
+    {
+      "epoch": 0.8637212196639701,
+      "grad_norm": 2.929445266723633,
+      "learning_rate": 3.989852398523986e-06,
+      "loss": 1.5799,
+      "step": 694
+    },
+    {
+      "epoch": 0.864965774735532,
+      "grad_norm": 2.593223810195923,
+      "learning_rate": 3.987546125461255e-06,
+      "loss": 1.5352,
+      "step": 695
+    },
+    {
+      "epoch": 0.866210329807094,
+      "grad_norm": 2.7710154056549072,
+      "learning_rate": 3.9852398523985245e-06,
+      "loss": 1.5048,
+      "step": 696
+    },
+    {
+      "epoch": 0.8674548848786559,
+      "grad_norm": 2.7370848655700684,
+      "learning_rate": 3.982933579335794e-06,
+      "loss": 1.5233,
+      "step": 697
+    },
+    {
+      "epoch": 0.8686994399502178,
+      "grad_norm": 2.8632168769836426,
+      "learning_rate": 3.980627306273063e-06,
+      "loss": 1.5435,
+      "step": 698
+    },
+    {
+      "epoch": 0.8699439950217797,
+      "grad_norm": 2.752298593521118,
+      "learning_rate": 3.978321033210332e-06,
+      "loss": 1.5728,
+      "step": 699
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "grad_norm": 2.8345625400543213,
+      "learning_rate": 3.976014760147602e-06,
+      "loss": 1.5727,
+      "step": 700
+    },
+    {
+      "epoch": 0.8711885500933416,
+      "eval_loss": 1.6855305433273315,
+      "eval_runtime": 43.3683,
+      "eval_samples_per_second": 23.058,
+      "eval_steps_per_second": 0.968,
+      "step": 700
+    },
+    {
+      "epoch": 0.8724331051649036,
+      "grad_norm": 2.8135335445404053,
+      "learning_rate": 3.973708487084872e-06,
+      "loss": 1.5086,
+      "step": 701
+    },
+    {
+      "epoch": 0.8736776602364654,
+      "grad_norm": 2.7416603565216064,
+      "learning_rate": 3.97140221402214e-06,
+      "loss": 1.545,
+      "step": 702
+    },
+    {
+      "epoch": 0.8749222153080274,
+      "grad_norm": 2.6902289390563965,
+      "learning_rate": 3.96909594095941e-06,
+      "loss": 1.5509,
+      "step": 703
+    },
+    {
+      "epoch": 0.8761667703795893,
+      "grad_norm": 2.6537084579467773,
+      "learning_rate": 3.966789667896679e-06,
+      "loss": 1.562,
+      "step": 704
+    },
+    {
+      "epoch": 0.8774113254511512,
+      "grad_norm": 2.6979284286499023,
+      "learning_rate": 3.964483394833948e-06,
+      "loss": 1.4829,
+      "step": 705
+    },
+    {
+      "epoch": 0.8786558805227132,
+      "grad_norm": 2.5795764923095703,
+      "learning_rate": 3.962177121771218e-06,
+      "loss": 1.4677,
+      "step": 706
+    },
+    {
+      "epoch": 0.879900435594275,
+      "grad_norm": 3.0185086727142334,
+      "learning_rate": 3.959870848708487e-06,
+      "loss": 1.5608,
+      "step": 707
+    },
+    {
+      "epoch": 0.881144990665837,
+      "grad_norm": 2.6458113193511963,
+      "learning_rate": 3.957564575645757e-06,
+      "loss": 1.5342,
+      "step": 708
+    },
+    {
+      "epoch": 0.8823895457373989,
+      "grad_norm": 2.7985665798187256,
+      "learning_rate": 3.955258302583026e-06,
+      "loss": 1.527,
+      "step": 709
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "grad_norm": 2.9069082736968994,
+      "learning_rate": 3.9529520295202955e-06,
+      "loss": 1.5156,
+      "step": 710
+    },
+    {
+      "epoch": 0.8836341008089608,
+      "eval_loss": 1.6898822784423828,
+      "eval_runtime": 42.8749,
+      "eval_samples_per_second": 23.324,
+      "eval_steps_per_second": 0.98,
+      "step": 710
+    },
+    {
+      "epoch": 0.8848786558805227,
+      "grad_norm": 2.760143995285034,
+      "learning_rate": 3.950645756457565e-06,
+      "loss": 1.5124,
+      "step": 711
+    },
+    {
+      "epoch": 0.8861232109520847,
+      "grad_norm": 2.8085529804229736,
+      "learning_rate": 3.948339483394834e-06,
+      "loss": 1.5862,
+      "step": 712
+    },
+    {
+      "epoch": 0.8873677660236465,
+      "grad_norm": 2.909905195236206,
+      "learning_rate": 3.946033210332104e-06,
+      "loss": 1.5557,
+      "step": 713
+    },
+    {
+      "epoch": 0.8886123210952085,
+      "grad_norm": 2.826899290084839,
+      "learning_rate": 3.943726937269373e-06,
+      "loss": 1.5277,
+      "step": 714
+    },
+    {
+      "epoch": 0.8898568761667703,
+      "grad_norm": 2.6269052028656006,
+      "learning_rate": 3.941420664206642e-06,
+      "loss": 1.4819,
+      "step": 715
+    },
+    {
+      "epoch": 0.8911014312383323,
+      "grad_norm": 2.6687698364257812,
+      "learning_rate": 3.939114391143912e-06,
+      "loss": 1.5156,
+      "step": 716
+    },
+    {
+      "epoch": 0.8923459863098943,
+      "grad_norm": 2.9605629444122314,
+      "learning_rate": 3.9368081180811815e-06,
+      "loss": 1.5358,
+      "step": 717
+    },
+    {
+      "epoch": 0.8935905413814561,
+      "grad_norm": 3.0367166996002197,
+      "learning_rate": 3.934501845018451e-06,
+      "loss": 1.5847,
+      "step": 718
+    },
+    {
+      "epoch": 0.8948350964530181,
+      "grad_norm": 2.528796434402466,
+      "learning_rate": 3.9321955719557195e-06,
+      "loss": 1.4851,
+      "step": 719
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "grad_norm": 2.7116613388061523,
+      "learning_rate": 3.929889298892989e-06,
+      "loss": 1.4706,
+      "step": 720
+    },
+    {
+      "epoch": 0.8960796515245799,
+      "eval_loss": 1.6845883131027222,
+      "eval_runtime": 44.4734,
+      "eval_samples_per_second": 22.485,
+      "eval_steps_per_second": 0.944,
+      "step": 720
+    },
+    {
+      "epoch": 0.8973242065961419,
+      "grad_norm": 2.771864414215088,
+      "learning_rate": 3.927583025830259e-06,
+      "loss": 1.5444,
+      "step": 721
+    },
+    {
+      "epoch": 0.8985687616677038,
+      "grad_norm": 2.773746967315674,
+      "learning_rate": 3.925276752767528e-06,
+      "loss": 1.564,
+      "step": 722
+    },
+    {
+      "epoch": 0.8998133167392657,
+      "grad_norm": 2.7069854736328125,
+      "learning_rate": 3.922970479704797e-06,
+      "loss": 1.5601,
+      "step": 723
+    },
+    {
+      "epoch": 0.9010578718108276,
+      "grad_norm": 2.840261697769165,
+      "learning_rate": 3.920664206642067e-06,
+      "loss": 1.5733,
+      "step": 724
+    },
+    {
+      "epoch": 0.9023024268823896,
+      "grad_norm": 2.709897041320801,
+      "learning_rate": 3.918357933579336e-06,
+      "loss": 1.528,
+      "step": 725
+    },
+    {
+      "epoch": 0.9035469819539514,
+      "grad_norm": 2.662367343902588,
+      "learning_rate": 3.9160516605166055e-06,
+      "loss": 1.5532,
+      "step": 726
+    },
+    {
+      "epoch": 0.9047915370255134,
+      "grad_norm": 2.6775717735290527,
+      "learning_rate": 3.913745387453875e-06,
+      "loss": 1.5464,
+      "step": 727
+    },
+    {
+      "epoch": 0.9060360920970753,
+      "grad_norm": 2.617842674255371,
+      "learning_rate": 3.911439114391144e-06,
+      "loss": 1.4539,
+      "step": 728
+    },
+    {
+      "epoch": 0.9072806471686372,
+      "grad_norm": 2.668649435043335,
+      "learning_rate": 3.909132841328414e-06,
+      "loss": 1.5627,
+      "step": 729
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "grad_norm": 2.5331950187683105,
+      "learning_rate": 3.906826568265683e-06,
+      "loss": 1.5165,
+      "step": 730
+    },
+    {
+      "epoch": 0.9085252022401992,
+      "eval_loss": 1.6860331296920776,
+      "eval_runtime": 46.094,
+      "eval_samples_per_second": 21.695,
+      "eval_steps_per_second": 0.911,
+      "step": 730
+    },
+    {
+      "epoch": 0.909769757311761,
+      "grad_norm": 2.602492332458496,
+      "learning_rate": 3.9045202952029526e-06,
+      "loss": 1.5145,
+      "step": 731
+    },
+    {
+      "epoch": 0.911014312383323,
+      "grad_norm": 2.632782459259033,
+      "learning_rate": 3.902214022140222e-06,
+      "loss": 1.4413,
+      "step": 732
+    },
+    {
+      "epoch": 0.9122588674548848,
+      "grad_norm": 2.6025912761688232,
+      "learning_rate": 3.899907749077491e-06,
+      "loss": 1.5263,
+      "step": 733
+    },
+    {
+      "epoch": 0.9135034225264468,
+      "grad_norm": 2.770116090774536,
+      "learning_rate": 3.897601476014761e-06,
+      "loss": 1.5514,
+      "step": 734
+    },
+    {
+      "epoch": 0.9147479775980087,
+      "grad_norm": 2.7822961807250977,
+      "learning_rate": 3.895295202952029e-06,
+      "loss": 1.5296,
+      "step": 735
+    },
+    {
+      "epoch": 0.9159925326695706,
+      "grad_norm": 2.6689720153808594,
+      "learning_rate": 3.892988929889299e-06,
+      "loss": 1.4552,
+      "step": 736
+    },
+    {
+      "epoch": 0.9172370877411326,
+      "grad_norm": 2.8021798133850098,
+      "learning_rate": 3.890682656826569e-06,
+      "loss": 1.5308,
+      "step": 737
+    },
+    {
+      "epoch": 0.9184816428126945,
+      "grad_norm": 2.575313091278076,
+      "learning_rate": 3.888376383763838e-06,
+      "loss": 1.4807,
+      "step": 738
+    },
+    {
+      "epoch": 0.9197261978842564,
+      "grad_norm": 2.5955779552459717,
+      "learning_rate": 3.886070110701107e-06,
+      "loss": 1.4222,
+      "step": 739
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "grad_norm": 2.647939920425415,
+      "learning_rate": 3.8837638376383765e-06,
+      "loss": 1.5285,
+      "step": 740
+    },
+    {
+      "epoch": 0.9209707529558183,
+      "eval_loss": 1.6907480955123901,
+      "eval_runtime": 45.7678,
+      "eval_samples_per_second": 21.849,
+      "eval_steps_per_second": 0.918,
+      "step": 740
+    },
+    {
+      "epoch": 0.9222153080273802,
+      "grad_norm": 2.7664523124694824,
+      "learning_rate": 3.881457564575646e-06,
+      "loss": 1.5349,
+      "step": 741
+    },
+    {
+      "epoch": 0.9234598630989421,
+      "grad_norm": 2.5770998001098633,
+      "learning_rate": 3.879151291512915e-06,
+      "loss": 1.5071,
+      "step": 742
+    },
+    {
+      "epoch": 0.9247044181705041,
+      "grad_norm": 2.502567768096924,
+      "learning_rate": 3.876845018450185e-06,
+      "loss": 1.492,
+      "step": 743
+    },
+    {
+      "epoch": 0.9259489732420659,
+      "grad_norm": 2.6531119346618652,
+      "learning_rate": 3.874538745387454e-06,
+      "loss": 1.5045,
+      "step": 744
+    },
+    {
+      "epoch": 0.9271935283136279,
+      "grad_norm": 2.7343876361846924,
+      "learning_rate": 3.872232472324724e-06,
+      "loss": 1.5194,
+      "step": 745
+    },
+    {
+      "epoch": 0.9284380833851898,
+      "grad_norm": 2.842313528060913,
+      "learning_rate": 3.869926199261993e-06,
+      "loss": 1.5588,
+      "step": 746
+    },
+    {
+      "epoch": 0.9296826384567517,
+      "grad_norm": 2.718245267868042,
+      "learning_rate": 3.8676199261992625e-06,
+      "loss": 1.5252,
+      "step": 747
+    },
+    {
+      "epoch": 0.9309271935283137,
+      "grad_norm": 2.695392370223999,
+      "learning_rate": 3.865313653136532e-06,
+      "loss": 1.5596,
+      "step": 748
+    },
+    {
+      "epoch": 0.9321717485998755,
+      "grad_norm": 2.7859959602355957,
+      "learning_rate": 3.863007380073801e-06,
+      "loss": 1.5151,
+      "step": 749
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "grad_norm": 2.5199174880981445,
+      "learning_rate": 3.860701107011071e-06,
+      "loss": 1.4948,
+      "step": 750
+    },
+    {
+      "epoch": 0.9334163036714375,
+      "eval_loss": 1.6868674755096436,
+      "eval_runtime": 46.4115,
+      "eval_samples_per_second": 21.546,
+      "eval_steps_per_second": 0.905,
+      "step": 750
+    },
+    {
+      "epoch": 0.9346608587429994,
+      "grad_norm": 2.8194046020507812,
+      "learning_rate": 3.858394833948339e-06,
+      "loss": 1.5144,
+      "step": 751
+    },
+    {
+      "epoch": 0.9359054138145613,
+      "grad_norm": 2.810380697250366,
+      "learning_rate": 3.856088560885609e-06,
+      "loss": 1.4998,
+      "step": 752
+    },
+    {
+      "epoch": 0.9371499688861232,
+      "grad_norm": 2.6906750202178955,
+      "learning_rate": 3.853782287822879e-06,
+      "loss": 1.4748,
+      "step": 753
+    },
+    {
+      "epoch": 0.9383945239576851,
+      "grad_norm": 2.9030277729034424,
+      "learning_rate": 3.851476014760148e-06,
+      "loss": 1.615,
+      "step": 754
+    },
+    {
+      "epoch": 0.939639079029247,
+      "grad_norm": 2.8690176010131836,
+      "learning_rate": 3.849169741697417e-06,
+      "loss": 1.5083,
+      "step": 755
+    },
+    {
+      "epoch": 0.940883634100809,
+      "grad_norm": 2.669646978378296,
+      "learning_rate": 3.8468634686346865e-06,
+      "loss": 1.5572,
+      "step": 756
+    },
+    {
+      "epoch": 0.9421281891723708,
+      "grad_norm": 2.6517696380615234,
+      "learning_rate": 3.844557195571956e-06,
+      "loss": 1.5142,
+      "step": 757
+    },
+    {
+      "epoch": 0.9433727442439328,
+      "grad_norm": 2.5524444580078125,
+      "learning_rate": 3.842250922509225e-06,
+      "loss": 1.4924,
+      "step": 758
+    },
+    {
+      "epoch": 0.9446172993154948,
+      "grad_norm": 2.6532633304595947,
+      "learning_rate": 3.839944649446495e-06,
+      "loss": 1.484,
+      "step": 759
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "grad_norm": 2.7779057025909424,
+      "learning_rate": 3.837638376383764e-06,
+      "loss": 1.5106,
+      "step": 760
+    },
+    {
+      "epoch": 0.9458618543870566,
+      "eval_loss": 1.6845752000808716,
+      "eval_runtime": 50.3415,
+      "eval_samples_per_second": 19.864,
+      "eval_steps_per_second": 0.834,
+      "step": 760
+    },
+    {
+      "epoch": 0.9471064094586186,
+      "grad_norm": 2.6192541122436523,
+      "learning_rate": 3.8353321033210336e-06,
+      "loss": 1.5454,
+      "step": 761
+    },
+    {
+      "epoch": 0.9483509645301804,
+      "grad_norm": 2.561861991882324,
+      "learning_rate": 3.833025830258303e-06,
+      "loss": 1.5141,
+      "step": 762
+    },
+    {
+      "epoch": 0.9495955196017424,
+      "grad_norm": 2.661829948425293,
+      "learning_rate": 3.830719557195572e-06,
+      "loss": 1.4997,
+      "step": 763
+    },
+    {
+      "epoch": 0.9508400746733043,
+      "grad_norm": 2.725275754928589,
+      "learning_rate": 3.828413284132842e-06,
+      "loss": 1.5224,
+      "step": 764
+    },
+    {
+      "epoch": 0.9520846297448662,
+      "grad_norm": 2.5664193630218506,
+      "learning_rate": 3.826107011070111e-06,
+      "loss": 1.5073,
+      "step": 765
+    },
+    {
+      "epoch": 0.9533291848164281,
+      "grad_norm": 2.595189332962036,
+      "learning_rate": 3.823800738007381e-06,
+      "loss": 1.5379,
+      "step": 766
+    },
+    {
+      "epoch": 0.95457373988799,
+      "grad_norm": 2.738060235977173,
+      "learning_rate": 3.821494464944649e-06,
+      "loss": 1.4814,
+      "step": 767
+    },
+    {
+      "epoch": 0.955818294959552,
+      "grad_norm": 2.601071357727051,
+      "learning_rate": 3.819188191881919e-06,
+      "loss": 1.4812,
+      "step": 768
+    },
+    {
+      "epoch": 0.9570628500311139,
+      "grad_norm": 2.657212257385254,
+      "learning_rate": 3.816881918819189e-06,
+      "loss": 1.4748,
+      "step": 769
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "grad_norm": 2.7673351764678955,
+      "learning_rate": 3.814575645756458e-06,
+      "loss": 1.5535,
+      "step": 770
+    },
+    {
+      "epoch": 0.9583074051026758,
+      "eval_loss": 1.677243947982788,
+      "eval_runtime": 51.5537,
+      "eval_samples_per_second": 19.397,
+      "eval_steps_per_second": 0.815,
+      "step": 770
+    },
+    {
+      "epoch": 0.9595519601742377,
+      "grad_norm": 2.760890007019043,
+      "learning_rate": 3.812269372693727e-06,
+      "loss": 1.5654,
+      "step": 771
+    },
+    {
+      "epoch": 0.9607965152457997,
+      "grad_norm": 2.6934309005737305,
+      "learning_rate": 3.809963099630997e-06,
+      "loss": 1.5149,
+      "step": 772
+    },
+    {
+      "epoch": 0.9620410703173615,
+      "grad_norm": 2.729950428009033,
+      "learning_rate": 3.8076568265682662e-06,
+      "loss": 1.5579,
+      "step": 773
+    },
+    {
+      "epoch": 0.9632856253889235,
+      "grad_norm": 2.732926607131958,
+      "learning_rate": 3.8053505535055352e-06,
+      "loss": 1.5179,
+      "step": 774
+    },
+    {
+      "epoch": 0.9645301804604853,
+      "grad_norm": 2.745391368865967,
+      "learning_rate": 3.8030442804428046e-06,
+      "loss": 1.5124,
+      "step": 775
+    },
+    {
+      "epoch": 0.9657747355320473,
+      "grad_norm": 2.8050146102905273,
+      "learning_rate": 3.800738007380074e-06,
+      "loss": 1.519,
+      "step": 776
+    },
+    {
+      "epoch": 0.9670192906036092,
+      "grad_norm": 2.697171449661255,
+      "learning_rate": 3.798431734317343e-06,
+      "loss": 1.5219,
+      "step": 777
+    },
+    {
+      "epoch": 0.9682638456751711,
+      "grad_norm": 2.6375980377197266,
+      "learning_rate": 3.796125461254613e-06,
+      "loss": 1.5345,
+      "step": 778
+    },
+    {
+      "epoch": 0.9695084007467331,
+      "grad_norm": 2.586636781692505,
+      "learning_rate": 3.7938191881918823e-06,
+      "loss": 1.5182,
+      "step": 779
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "grad_norm": 2.626453399658203,
+      "learning_rate": 3.7915129151291518e-06,
+      "loss": 1.4817,
+      "step": 780
+    },
+    {
+      "epoch": 0.9707529558182949,
+      "eval_loss": 1.6734713315963745,
+      "eval_runtime": 48.1436,
+      "eval_samples_per_second": 20.771,
+      "eval_steps_per_second": 0.872,
+      "step": 780
+    },
+    {
+      "epoch": 0.9719975108898569,
+      "grad_norm": 2.579970121383667,
+      "learning_rate": 3.7892066420664208e-06,
+      "loss": 1.4981,
+      "step": 781
+    },
+    {
+      "epoch": 0.9732420659614188,
+      "grad_norm": 2.6834843158721924,
+      "learning_rate": 3.7869003690036906e-06,
+      "loss": 1.5044,
+      "step": 782
+    },
+    {
+      "epoch": 0.9744866210329807,
+      "grad_norm": 2.586982488632202,
+      "learning_rate": 3.78459409594096e-06,
+      "loss": 1.5079,
+      "step": 783
+    },
+    {
+      "epoch": 0.9757311761045426,
+      "grad_norm": 2.910027503967285,
+      "learning_rate": 3.782287822878229e-06,
+      "loss": 1.5695,
+      "step": 784
+    },
+    {
+      "epoch": 0.9769757311761046,
+      "grad_norm": 2.7088494300842285,
+      "learning_rate": 3.7799815498154984e-06,
+      "loss": 1.4986,
+      "step": 785
+    },
+    {
+      "epoch": 0.9782202862476664,
+      "grad_norm": 2.581325054168701,
+      "learning_rate": 3.777675276752768e-06,
+      "loss": 1.5137,
+      "step": 786
+    },
+    {
+      "epoch": 0.9794648413192284,
+      "grad_norm": 2.700709819793701,
+      "learning_rate": 3.775369003690037e-06,
+      "loss": 1.4718,
+      "step": 787
+    },
+    {
+      "epoch": 0.9807093963907902,
+      "grad_norm": 2.6712708473205566,
+      "learning_rate": 3.7730627306273067e-06,
+      "loss": 1.5208,
+      "step": 788
+    },
+    {
+      "epoch": 0.9819539514623522,
+      "grad_norm": 2.6799817085266113,
+      "learning_rate": 3.770756457564576e-06,
+      "loss": 1.5374,
+      "step": 789
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "grad_norm": 2.618988037109375,
+      "learning_rate": 3.768450184501845e-06,
+      "loss": 1.4997,
+      "step": 790
+    },
+    {
+      "epoch": 0.9831985065339142,
+      "eval_loss": 1.6860820055007935,
+      "eval_runtime": 51.7654,
+      "eval_samples_per_second": 19.318,
+      "eval_steps_per_second": 0.811,
+      "step": 790
+    },
+    {
+      "epoch": 0.984443061605476,
+      "grad_norm": 2.6899526119232178,
+      "learning_rate": 3.7661439114391146e-06,
+      "loss": 1.5216,
+      "step": 791
+    },
+    {
+      "epoch": 0.985687616677038,
+      "grad_norm": 2.8003487586975098,
+      "learning_rate": 3.763837638376384e-06,
+      "loss": 1.557,
+      "step": 792
+    },
+    {
+      "epoch": 0.9869321717485999,
+      "grad_norm": 2.744536876678467,
+      "learning_rate": 3.761531365313654e-06,
+      "loss": 1.5355,
+      "step": 793
+    },
+    {
+      "epoch": 0.9881767268201618,
+      "grad_norm": 2.587250232696533,
+      "learning_rate": 3.759225092250923e-06,
+      "loss": 1.5259,
+      "step": 794
+    },
+    {
+      "epoch": 0.9894212818917237,
+      "grad_norm": 2.616291046142578,
+      "learning_rate": 3.7569188191881922e-06,
+      "loss": 1.5176,
+      "step": 795
+    },
+    {
+      "epoch": 0.9906658369632856,
+      "grad_norm": 2.6410577297210693,
+      "learning_rate": 3.7546125461254617e-06,
+      "loss": 1.511,
+      "step": 796
+    },
+    {
+      "epoch": 0.9919103920348475,
+      "grad_norm": 2.577373504638672,
+      "learning_rate": 3.7523062730627307e-06,
+      "loss": 1.4715,
+      "step": 797
+    },
+    {
+      "epoch": 0.9931549471064095,
+      "grad_norm": 2.67305326461792,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 1.4828,
+      "step": 798
+    },
+    {
+      "epoch": 0.9943995021779714,
+      "grad_norm": 3.1631500720977783,
+      "learning_rate": 3.74769372693727e-06,
+      "loss": 1.5602,
+      "step": 799
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "grad_norm": 2.9222350120544434,
+      "learning_rate": 3.745387453874539e-06,
+      "loss": 1.4814,
+      "step": 800
+    },
+    {
+      "epoch": 0.9956440572495333,
+      "eval_loss": 1.6810544729232788,
+      "eval_runtime": 51.1262,
+      "eval_samples_per_second": 19.559,
+      "eval_steps_per_second": 0.821,
+      "step": 800
+    },
+    {
+      "epoch": 0.9968886123210952,
+      "grad_norm": 2.884312629699707,
+      "learning_rate": 3.7430811808118084e-06,
+      "loss": 1.4553,
+      "step": 801
+    },
+    {
+      "epoch": 0.9981331673926571,
+      "grad_norm": 2.712655782699585,
+      "learning_rate": 3.7407749077490778e-06,
+      "loss": 1.5275,
+      "step": 802
+    },
+    {
+      "epoch": 0.9993777224642191,
+      "grad_norm": 2.8700571060180664,
+      "learning_rate": 3.7384686346863468e-06,
+      "loss": 1.4923,
+      "step": 803
+    },
+    {
+      "epoch": 1.000622277535781,
+      "grad_norm": 2.7779245376586914,
+      "learning_rate": 3.7361623616236166e-06,
+      "loss": 1.5224,
+      "step": 804
+    },
+    {
+      "epoch": 1.0018668326073428,
+      "grad_norm": 2.7679810523986816,
+      "learning_rate": 3.733856088560886e-06,
+      "loss": 1.4977,
+      "step": 805
+    },
+    {
+      "epoch": 1.0031113876789048,
+      "grad_norm": 2.797023296356201,
+      "learning_rate": 3.7315498154981555e-06,
+      "loss": 1.4396,
+      "step": 806
+    },
+    {
+      "epoch": 1.0043559427504667,
+      "grad_norm": 3.007962465286255,
+      "learning_rate": 3.7292435424354245e-06,
+      "loss": 1.5015,
+      "step": 807
+    },
+    {
+      "epoch": 1.0056004978220285,
+      "grad_norm": 3.127639055252075,
+      "learning_rate": 3.726937269372694e-06,
+      "loss": 1.5176,
+      "step": 808
+    },
+    {
+      "epoch": 1.0068450528935906,
+      "grad_norm": 2.9001357555389404,
+      "learning_rate": 3.7246309963099637e-06,
+      "loss": 1.4735,
+      "step": 809
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "grad_norm": 2.8245413303375244,
+      "learning_rate": 3.7223247232472327e-06,
+      "loss": 1.486,
+      "step": 810
+    },
+    {
+      "epoch": 1.0080896079651525,
+      "eval_loss": 1.6835130453109741,
+      "eval_runtime": 44.5682,
+      "eval_samples_per_second": 22.438,
+      "eval_steps_per_second": 0.942,
+      "step": 810
+    },
+    {
+      "epoch": 1.0093341630367143,
+      "grad_norm": 2.780172824859619,
+      "learning_rate": 3.720018450184502e-06,
+      "loss": 1.4437,
+      "step": 811
+    },
+    {
+      "epoch": 1.0105787181082764,
+      "grad_norm": 2.6013996601104736,
+      "learning_rate": 3.7177121771217716e-06,
+      "loss": 1.4529,
+      "step": 812
+    },
+    {
+      "epoch": 1.0118232731798382,
+      "grad_norm": 2.831015110015869,
+      "learning_rate": 3.7154059040590406e-06,
+      "loss": 1.4825,
+      "step": 813
+    },
+    {
+      "epoch": 1.0130678282514,
+      "grad_norm": 2.8341829776763916,
+      "learning_rate": 3.71309963099631e-06,
+      "loss": 1.4736,
+      "step": 814
+    },
+    {
+      "epoch": 1.0143123833229621,
+      "grad_norm": 2.769540309906006,
+      "learning_rate": 3.71079335793358e-06,
+      "loss": 1.4724,
+      "step": 815
+    },
+    {
+      "epoch": 1.015556938394524,
+      "grad_norm": 3.0083069801330566,
+      "learning_rate": 3.708487084870849e-06,
+      "loss": 1.4985,
+      "step": 816
+    },
+    {
+      "epoch": 1.0168014934660858,
+      "grad_norm": 3.052168607711792,
+      "learning_rate": 3.7061808118081183e-06,
+      "loss": 1.5193,
+      "step": 817
+    },
+    {
+      "epoch": 1.0180460485376477,
+      "grad_norm": 2.924877405166626,
+      "learning_rate": 3.7038745387453877e-06,
+      "loss": 1.5219,
+      "step": 818
+    },
+    {
+      "epoch": 1.0192906036092098,
+      "grad_norm": 3.150123357772827,
+      "learning_rate": 3.7015682656826576e-06,
+      "loss": 1.5026,
+      "step": 819
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "grad_norm": 2.881655216217041,
+      "learning_rate": 3.6992619926199266e-06,
+      "loss": 1.5011,
+      "step": 820
+    },
+    {
+      "epoch": 1.0205351586807716,
+      "eval_loss": 1.6745303869247437,
+      "eval_runtime": 50.6791,
+      "eval_samples_per_second": 19.732,
+      "eval_steps_per_second": 0.829,
+      "step": 820
+    },
+    {
+      "epoch": 1.0217797137523335,
+      "grad_norm": 2.824249505996704,
+      "learning_rate": 3.696955719557196e-06,
+      "loss": 1.4642,
+      "step": 821
+    },
+    {
+      "epoch": 1.0230242688238955,
+      "grad_norm": 2.859144926071167,
+      "learning_rate": 3.6946494464944654e-06,
+      "loss": 1.4681,
+      "step": 822
+    },
+    {
+      "epoch": 1.0242688238954574,
+      "grad_norm": 2.90312123298645,
+      "learning_rate": 3.6923431734317344e-06,
+      "loss": 1.4926,
+      "step": 823
+    },
+    {
+      "epoch": 1.0255133789670192,
+      "grad_norm": 2.8340659141540527,
+      "learning_rate": 3.690036900369004e-06,
+      "loss": 1.4834,
+      "step": 824
+    },
+    {
+      "epoch": 1.0267579340385813,
+      "grad_norm": 2.9743151664733887,
+      "learning_rate": 3.6877306273062737e-06,
+      "loss": 1.4574,
+      "step": 825
+    },
+    {
+      "epoch": 1.0280024891101431,
+      "grad_norm": 3.032179594039917,
+      "learning_rate": 3.6854243542435427e-06,
+      "loss": 1.5571,
+      "step": 826
+    },
+    {
+      "epoch": 1.029247044181705,
+      "grad_norm": 2.816826343536377,
+      "learning_rate": 3.683118081180812e-06,
+      "loss": 1.455,
+      "step": 827
+    },
+    {
+      "epoch": 1.030491599253267,
+      "grad_norm": 3.046696186065674,
+      "learning_rate": 3.6808118081180815e-06,
+      "loss": 1.4871,
+      "step": 828
+    },
+    {
+      "epoch": 1.031736154324829,
+      "grad_norm": 2.8517391681671143,
+      "learning_rate": 3.678505535055351e-06,
+      "loss": 1.5292,
+      "step": 829
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "grad_norm": 2.752811908721924,
+      "learning_rate": 3.67619926199262e-06,
+      "loss": 1.5091,
+      "step": 830
+    },
+    {
+      "epoch": 1.0329807093963908,
+      "eval_loss": 1.6692287921905518,
+      "eval_runtime": 50.8347,
+      "eval_samples_per_second": 19.672,
+      "eval_steps_per_second": 0.826,
+      "step": 830
+    },
+    {
+      "epoch": 1.0342252644679526,
+      "grad_norm": 2.9220356941223145,
+      "learning_rate": 3.6738929889298898e-06,
+      "loss": 1.4247,
+      "step": 831
+    },
+    {
+      "epoch": 1.0354698195395147,
+      "grad_norm": 2.8903002738952637,
+      "learning_rate": 3.671586715867159e-06,
+      "loss": 1.5595,
+      "step": 832
+    },
+    {
+      "epoch": 1.0367143746110765,
+      "grad_norm": 2.6495542526245117,
+      "learning_rate": 3.669280442804428e-06,
+      "loss": 1.4076,
+      "step": 833
+    },
+    {
+      "epoch": 1.0379589296826384,
+      "grad_norm": 2.880809783935547,
+      "learning_rate": 3.6669741697416976e-06,
+      "loss": 1.5285,
+      "step": 834
+    },
+    {
+      "epoch": 1.0392034847542004,
+      "grad_norm": 2.919142007827759,
+      "learning_rate": 3.6646678966789675e-06,
+      "loss": 1.5165,
+      "step": 835
+    },
+    {
+      "epoch": 1.0404480398257623,
+      "grad_norm": 3.1180522441864014,
+      "learning_rate": 3.6623616236162365e-06,
+      "loss": 1.5224,
+      "step": 836
+    },
+    {
+      "epoch": 1.0416925948973241,
+      "grad_norm": 2.767709255218506,
+      "learning_rate": 3.660055350553506e-06,
+      "loss": 1.4859,
+      "step": 837
+    },
+    {
+      "epoch": 1.0429371499688862,
+      "grad_norm": 2.708828926086426,
+      "learning_rate": 3.6577490774907753e-06,
+      "loss": 1.4803,
+      "step": 838
+    },
+    {
+      "epoch": 1.044181705040448,
+      "grad_norm": 2.688359260559082,
+      "learning_rate": 3.6554428044280443e-06,
+      "loss": 1.4571,
+      "step": 839
+    },
+    {
+      "epoch": 1.04542626011201,
+      "grad_norm": 2.691812753677368,
+      "learning_rate": 3.6531365313653137e-06,
+      "loss": 1.5043,
+      "step": 840
+    },
+    {
+      "epoch": 1.04542626011201,
+      "eval_loss": 1.6769559383392334,
+      "eval_runtime": 49.6233,
+      "eval_samples_per_second": 20.152,
+      "eval_steps_per_second": 0.846,
+      "step": 840
+    },
+    {
+      "epoch": 1.046670815183572,
+      "grad_norm": 2.686372995376587,
+      "learning_rate": 3.6508302583025836e-06,
+      "loss": 1.5221,
+      "step": 841
+    },
+    {
+      "epoch": 1.0479153702551338,
+      "grad_norm": 2.742859125137329,
+      "learning_rate": 3.648523985239853e-06,
+      "loss": 1.4685,
+      "step": 842
+    },
+    {
+      "epoch": 1.0491599253266957,
+      "grad_norm": 2.874018907546997,
+      "learning_rate": 3.646217712177122e-06,
+      "loss": 1.4813,
+      "step": 843
+    },
+    {
+      "epoch": 1.0504044803982575,
+      "grad_norm": 2.8159990310668945,
+      "learning_rate": 3.6439114391143914e-06,
+      "loss": 1.4421,
+      "step": 844
+    },
+    {
+      "epoch": 1.0516490354698196,
+      "grad_norm": 2.714787006378174,
+      "learning_rate": 3.641605166051661e-06,
+      "loss": 1.4879,
+      "step": 845
+    },
+    {
+      "epoch": 1.0528935905413814,
+      "grad_norm": 2.885148525238037,
+      "learning_rate": 3.63929889298893e-06,
+      "loss": 1.4725,
+      "step": 846
+    },
+    {
+      "epoch": 1.0541381456129433,
+      "grad_norm": 2.7272610664367676,
+      "learning_rate": 3.6369926199261997e-06,
+      "loss": 1.4276,
+      "step": 847
+    },
+    {
+      "epoch": 1.0553827006845053,
+      "grad_norm": 2.9114978313446045,
+      "learning_rate": 3.634686346863469e-06,
+      "loss": 1.4194,
+      "step": 848
+    },
+    {
+      "epoch": 1.0566272557560672,
+      "grad_norm": 3.019313335418701,
+      "learning_rate": 3.632380073800738e-06,
+      "loss": 1.5266,
+      "step": 849
+    },
+    {
+      "epoch": 1.057871810827629,
+      "grad_norm": 2.714751958847046,
+      "learning_rate": 3.6300738007380075e-06,
+      "loss": 1.4593,
+      "step": 850
+    },
+    {
+      "epoch": 1.057871810827629,
+      "eval_loss": 1.6760298013687134,
+      "eval_runtime": 52.1681,
+      "eval_samples_per_second": 19.169,
+      "eval_steps_per_second": 0.805,
+      "step": 850
+    },
+    {
+      "epoch": 1.0591163658991911,
+      "grad_norm": 3.0507936477661133,
+      "learning_rate": 3.627767527675277e-06,
+      "loss": 1.5064,
+      "step": 851
+    },
+    {
+      "epoch": 1.060360920970753,
+      "grad_norm": 2.8116295337677,
+      "learning_rate": 3.625461254612546e-06,
+      "loss": 1.5218,
+      "step": 852
+    },
+    {
+      "epoch": 1.0616054760423148,
+      "grad_norm": 2.8801465034484863,
+      "learning_rate": 3.623154981549816e-06,
+      "loss": 1.4641,
+      "step": 853
+    },
+    {
+      "epoch": 1.0628500311138769,
+      "grad_norm": 3.0581862926483154,
+      "learning_rate": 3.6208487084870852e-06,
+      "loss": 1.483,
+      "step": 854
+    },
+    {
+      "epoch": 1.0640945861854387,
+      "grad_norm": 2.898780584335327,
+      "learning_rate": 3.6185424354243547e-06,
+      "loss": 1.4623,
+      "step": 855
+    },
+    {
+      "epoch": 1.0653391412570006,
+      "grad_norm": 2.971820116043091,
+      "learning_rate": 3.6162361623616237e-06,
+      "loss": 1.4455,
+      "step": 856
+    },
+    {
+      "epoch": 1.0665836963285624,
+      "grad_norm": 3.0021913051605225,
+      "learning_rate": 3.6139298892988935e-06,
+      "loss": 1.4878,
+      "step": 857
+    },
+    {
+      "epoch": 1.0678282514001245,
+      "grad_norm": 3.0697457790374756,
+      "learning_rate": 3.611623616236163e-06,
+      "loss": 1.5008,
+      "step": 858
+    },
+    {
+      "epoch": 1.0690728064716863,
+      "grad_norm": 3.0787923336029053,
+      "learning_rate": 3.609317343173432e-06,
+      "loss": 1.4612,
+      "step": 859
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "grad_norm": 2.789339780807495,
+      "learning_rate": 3.6070110701107014e-06,
+      "loss": 1.4287,
+      "step": 860
+    },
+    {
+      "epoch": 1.0703173615432482,
+      "eval_loss": 1.6747506856918335,
+      "eval_runtime": 51.4912,
+      "eval_samples_per_second": 19.421,
+      "eval_steps_per_second": 0.816,
+      "step": 860
+    },
+    {
+      "epoch": 1.0715619166148103,
+      "grad_norm": 2.9622788429260254,
+      "learning_rate": 3.6047047970479708e-06,
+      "loss": 1.4905,
+      "step": 861
+    },
+    {
+      "epoch": 1.072806471686372,
+      "grad_norm": 3.0675268173217773,
+      "learning_rate": 3.6023985239852398e-06,
+      "loss": 1.461,
+      "step": 862
+    },
+    {
+      "epoch": 1.074051026757934,
+      "grad_norm": 2.820996046066284,
+      "learning_rate": 3.6000922509225096e-06,
+      "loss": 1.4948,
+      "step": 863
+    },
+    {
+      "epoch": 1.075295581829496,
+      "grad_norm": 2.833761215209961,
+      "learning_rate": 3.597785977859779e-06,
+      "loss": 1.4683,
+      "step": 864
+    },
+    {
+      "epoch": 1.0765401369010579,
+      "grad_norm": 2.917313814163208,
+      "learning_rate": 3.595479704797048e-06,
+      "loss": 1.5191,
+      "step": 865
+    },
+    {
+      "epoch": 1.0777846919726197,
+      "grad_norm": 2.876166820526123,
+      "learning_rate": 3.5931734317343175e-06,
+      "loss": 1.4854,
+      "step": 866
+    },
+    {
+      "epoch": 1.0790292470441818,
+      "grad_norm": 3.139049768447876,
+      "learning_rate": 3.590867158671587e-06,
+      "loss": 1.5134,
+      "step": 867
+    },
+    {
+      "epoch": 1.0802738021157436,
+      "grad_norm": 2.9956753253936768,
+      "learning_rate": 3.5885608856088567e-06,
+      "loss": 1.5003,
+      "step": 868
+    },
+    {
+      "epoch": 1.0815183571873055,
+      "grad_norm": 2.957181930541992,
+      "learning_rate": 3.5862546125461257e-06,
+      "loss": 1.4588,
+      "step": 869
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "grad_norm": 2.8576438426971436,
+      "learning_rate": 3.583948339483395e-06,
+      "loss": 1.48,
+      "step": 870
+    },
+    {
+      "epoch": 1.0827629122588673,
+      "eval_loss": 1.6724848747253418,
+      "eval_runtime": 50.9669,
+      "eval_samples_per_second": 19.621,
+      "eval_steps_per_second": 0.824,
+      "step": 870
+    },
+    {
+      "epoch": 1.0840074673304294,
+      "grad_norm": 2.8183560371398926,
+      "learning_rate": 3.5816420664206646e-06,
+      "loss": 1.4818,
+      "step": 871
+    },
+    {
+      "epoch": 1.0852520224019913,
+      "grad_norm": 2.7531797885894775,
+      "learning_rate": 3.5793357933579336e-06,
+      "loss": 1.4712,
+      "step": 872
+    },
+    {
+      "epoch": 1.086496577473553,
+      "grad_norm": 2.923962354660034,
+      "learning_rate": 3.5770295202952034e-06,
+      "loss": 1.4864,
+      "step": 873
+    },
+    {
+      "epoch": 1.0877411325451152,
+      "grad_norm": 2.851724147796631,
+      "learning_rate": 3.574723247232473e-06,
+      "loss": 1.4771,
+      "step": 874
+    },
+    {
+      "epoch": 1.088985687616677,
+      "grad_norm": 3.024634838104248,
+      "learning_rate": 3.572416974169742e-06,
+      "loss": 1.4643,
+      "step": 875
+    },
+    {
+      "epoch": 1.0902302426882389,
+      "grad_norm": 2.7100844383239746,
+      "learning_rate": 3.5701107011070113e-06,
+      "loss": 1.4773,
+      "step": 876
+    },
+    {
+      "epoch": 1.091474797759801,
+      "grad_norm": 2.730502128601074,
+      "learning_rate": 3.5678044280442807e-06,
+      "loss": 1.5051,
+      "step": 877
+    },
+    {
+      "epoch": 1.0927193528313628,
+      "grad_norm": 2.747082233428955,
+      "learning_rate": 3.5654981549815497e-06,
+      "loss": 1.5207,
+      "step": 878
+    },
+    {
+      "epoch": 1.0939639079029246,
+      "grad_norm": 2.718358039855957,
+      "learning_rate": 3.5631918819188195e-06,
+      "loss": 1.5067,
+      "step": 879
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "grad_norm": 2.7794573307037354,
+      "learning_rate": 3.560885608856089e-06,
+      "loss": 1.4616,
+      "step": 880
+    },
+    {
+      "epoch": 1.0952084629744867,
+      "eval_loss": 1.6676132678985596,
+      "eval_runtime": 51.6157,
+      "eval_samples_per_second": 19.374,
+      "eval_steps_per_second": 0.814,
+      "step": 880
+    },
+    {
+      "epoch": 1.0964530180460486,
+      "grad_norm": 2.727416753768921,
+      "learning_rate": 3.5585793357933584e-06,
+      "loss": 1.4043,
+      "step": 881
+    },
+    {
+      "epoch": 1.0976975731176104,
+      "grad_norm": 2.856900930404663,
+      "learning_rate": 3.5562730627306274e-06,
+      "loss": 1.5198,
+      "step": 882
+    },
+    {
+      "epoch": 1.0989421281891725,
+      "grad_norm": 2.872823715209961,
+      "learning_rate": 3.553966789667897e-06,
+      "loss": 1.4981,
+      "step": 883
+    },
+    {
+      "epoch": 1.1001866832607343,
+      "grad_norm": 2.7724361419677734,
+      "learning_rate": 3.5516605166051667e-06,
+      "loss": 1.4819,
+      "step": 884
+    },
+    {
+      "epoch": 1.1014312383322962,
+      "grad_norm": 2.884434461593628,
+      "learning_rate": 3.5493542435424357e-06,
+      "loss": 1.433,
+      "step": 885
+    },
+    {
+      "epoch": 1.102675793403858,
+      "grad_norm": 2.887873411178589,
+      "learning_rate": 3.547047970479705e-06,
+      "loss": 1.4677,
+      "step": 886
+    },
+    {
+      "epoch": 1.10392034847542,
+      "grad_norm": 2.946845054626465,
+      "learning_rate": 3.5447416974169745e-06,
+      "loss": 1.4983,
+      "step": 887
+    },
+    {
+      "epoch": 1.105164903546982,
+      "grad_norm": 2.752552032470703,
+      "learning_rate": 3.5424354243542435e-06,
+      "loss": 1.4766,
+      "step": 888
+    },
+    {
+      "epoch": 1.1064094586185438,
+      "grad_norm": 2.788634777069092,
+      "learning_rate": 3.5401291512915133e-06,
+      "loss": 1.4747,
+      "step": 889
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "grad_norm": 2.782792568206787,
+      "learning_rate": 3.5378228782287828e-06,
+      "loss": 1.5503,
+      "step": 890
+    },
+    {
+      "epoch": 1.1076540136901059,
+      "eval_loss": 1.6698333024978638,
+      "eval_runtime": 49.1373,
+      "eval_samples_per_second": 20.351,
+      "eval_steps_per_second": 0.855,
+      "step": 890
+    },
+    {
+      "epoch": 1.1088985687616677,
+      "grad_norm": 2.8351166248321533,
+      "learning_rate": 3.535516605166052e-06,
+      "loss": 1.4589,
+      "step": 891
+    },
+    {
+      "epoch": 1.1101431238332296,
+      "grad_norm": 2.798872709274292,
+      "learning_rate": 3.533210332103321e-06,
+      "loss": 1.4496,
+      "step": 892
+    },
+    {
+      "epoch": 1.1113876789047916,
+      "grad_norm": 3.0863239765167236,
+      "learning_rate": 3.5309040590405906e-06,
+      "loss": 1.4698,
+      "step": 893
+    },
+    {
+      "epoch": 1.1126322339763535,
+      "grad_norm": 2.853862762451172,
+      "learning_rate": 3.5285977859778605e-06,
+      "loss": 1.4429,
+      "step": 894
+    },
+    {
+      "epoch": 1.1138767890479153,
+      "grad_norm": 2.8946170806884766,
+      "learning_rate": 3.5262915129151295e-06,
+      "loss": 1.4697,
+      "step": 895
+    },
+    {
+      "epoch": 1.1151213441194774,
+      "grad_norm": 3.043823003768921,
+      "learning_rate": 3.523985239852399e-06,
+      "loss": 1.4666,
+      "step": 896
+    },
+    {
+      "epoch": 1.1163658991910392,
+      "grad_norm": 2.7822980880737305,
+      "learning_rate": 3.5216789667896683e-06,
+      "loss": 1.4523,
+      "step": 897
+    },
+    {
+      "epoch": 1.117610454262601,
+      "grad_norm": 2.9454171657562256,
+      "learning_rate": 3.5193726937269373e-06,
+      "loss": 1.5009,
+      "step": 898
+    },
+    {
+      "epoch": 1.1188550093341632,
+      "grad_norm": 2.8533740043640137,
+      "learning_rate": 3.5170664206642067e-06,
+      "loss": 1.4265,
+      "step": 899
+    },
+    {
+      "epoch": 1.120099564405725,
+      "grad_norm": 2.894937038421631,
+      "learning_rate": 3.5147601476014766e-06,
+      "loss": 1.4895,
+      "step": 900
+    },
+    {
+      "epoch": 1.120099564405725,
+      "eval_loss": 1.6701407432556152,
+      "eval_runtime": 45.477,
+      "eval_samples_per_second": 21.989,
+      "eval_steps_per_second": 0.924,
+      "step": 900
+    },
+    {
+      "epoch": 1.1213441194772868,
+      "grad_norm": 2.933821201324463,
+      "learning_rate": 3.5124538745387456e-06,
+      "loss": 1.468,
+      "step": 901
+    },
+    {
+      "epoch": 1.1225886745488487,
+      "grad_norm": 2.834265947341919,
+      "learning_rate": 3.510147601476015e-06,
+      "loss": 1.4621,
+      "step": 902
+    },
+    {
+      "epoch": 1.1238332296204108,
+      "grad_norm": 2.9064722061157227,
+      "learning_rate": 3.5078413284132844e-06,
+      "loss": 1.504,
+      "step": 903
+    },
+    {
+      "epoch": 1.1250777846919726,
+      "grad_norm": 2.8753886222839355,
+      "learning_rate": 3.505535055350554e-06,
+      "loss": 1.4643,
+      "step": 904
+    },
+    {
+      "epoch": 1.1263223397635345,
+      "grad_norm": 2.894659996032715,
+      "learning_rate": 3.503228782287823e-06,
+      "loss": 1.4909,
+      "step": 905
+    },
+    {
+      "epoch": 1.1275668948350965,
+      "grad_norm": 3.085977792739868,
+      "learning_rate": 3.5009225092250927e-06,
+      "loss": 1.459,
+      "step": 906
+    },
+    {
+      "epoch": 1.1288114499066584,
+      "grad_norm": 2.9958064556121826,
+      "learning_rate": 3.498616236162362e-06,
+      "loss": 1.4152,
+      "step": 907
+    },
+    {
+      "epoch": 1.1300560049782202,
+      "grad_norm": 2.9136762619018555,
+      "learning_rate": 3.496309963099631e-06,
+      "loss": 1.473,
+      "step": 908
+    },
+    {
+      "epoch": 1.1313005600497823,
+      "grad_norm": 2.956437826156616,
+      "learning_rate": 3.4940036900369005e-06,
+      "loss": 1.5167,
+      "step": 909
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "grad_norm": 2.7354865074157715,
+      "learning_rate": 3.4916974169741704e-06,
+      "loss": 1.4413,
+      "step": 910
+    },
+    {
+      "epoch": 1.1325451151213441,
+      "eval_loss": 1.6629912853240967,
+      "eval_runtime": 41.5478,
+      "eval_samples_per_second": 24.069,
+      "eval_steps_per_second": 1.011,
+      "step": 910
+    },
+    {
+      "epoch": 1.133789670192906,
+      "grad_norm": 2.7443008422851562,
+      "learning_rate": 3.4893911439114394e-06,
+      "loss": 1.4475,
+      "step": 911
+    },
+    {
+      "epoch": 1.135034225264468,
+      "grad_norm": 2.7811238765716553,
+      "learning_rate": 3.487084870848709e-06,
+      "loss": 1.4257,
+      "step": 912
+    },
+    {
+      "epoch": 1.13627878033603,
+      "grad_norm": 2.7916479110717773,
+      "learning_rate": 3.4847785977859782e-06,
+      "loss": 1.4048,
+      "step": 913
+    },
+    {
+      "epoch": 1.1375233354075918,
+      "grad_norm": 2.7835633754730225,
+      "learning_rate": 3.4824723247232472e-06,
+      "loss": 1.3996,
+      "step": 914
+    },
+    {
+      "epoch": 1.1387678904791536,
+      "grad_norm": 2.8188705444335938,
+      "learning_rate": 3.4801660516605166e-06,
+      "loss": 1.4541,
+      "step": 915
+    },
+    {
+      "epoch": 1.1400124455507157,
+      "grad_norm": 3.0134100914001465,
+      "learning_rate": 3.4778597785977865e-06,
+      "loss": 1.4353,
+      "step": 916
+    },
+    {
+      "epoch": 1.1412570006222775,
+      "grad_norm": 2.6965606212615967,
+      "learning_rate": 3.475553505535056e-06,
+      "loss": 1.4308,
+      "step": 917
+    },
+    {
+      "epoch": 1.1425015556938394,
+      "grad_norm": 2.8179919719696045,
+      "learning_rate": 3.473247232472325e-06,
+      "loss": 1.5204,
+      "step": 918
+    },
+    {
+      "epoch": 1.1437461107654014,
+      "grad_norm": 2.7213902473449707,
+      "learning_rate": 3.4709409594095943e-06,
+      "loss": 1.4344,
+      "step": 919
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "grad_norm": 2.850193738937378,
+      "learning_rate": 3.4686346863468638e-06,
+      "loss": 1.473,
+      "step": 920
+    },
+    {
+      "epoch": 1.1449906658369633,
+      "eval_loss": 1.663345217704773,
+      "eval_runtime": 43.8397,
+      "eval_samples_per_second": 22.81,
+      "eval_steps_per_second": 0.958,
+      "step": 920
+    },
+    {
+      "epoch": 1.1462352209085251,
+      "grad_norm": 2.780198574066162,
+      "learning_rate": 3.4663284132841328e-06,
+      "loss": 1.4472,
+      "step": 921
+    },
+    {
+      "epoch": 1.1474797759800872,
+      "grad_norm": 2.9373831748962402,
+      "learning_rate": 3.4640221402214026e-06,
+      "loss": 1.426,
+      "step": 922
+    },
+    {
+      "epoch": 1.148724331051649,
+      "grad_norm": 2.6986873149871826,
+      "learning_rate": 3.461715867158672e-06,
+      "loss": 1.3953,
+      "step": 923
+    },
+    {
+      "epoch": 1.149968886123211,
+      "grad_norm": 2.927441120147705,
+      "learning_rate": 3.459409594095941e-06,
+      "loss": 1.4578,
+      "step": 924
+    },
+    {
+      "epoch": 1.151213441194773,
+      "grad_norm": 2.8469648361206055,
+      "learning_rate": 3.4571033210332105e-06,
+      "loss": 1.4313,
+      "step": 925
+    },
+    {
+      "epoch": 1.1524579962663348,
+      "grad_norm": 2.8114013671875,
+      "learning_rate": 3.4547970479704803e-06,
+      "loss": 1.4267,
+      "step": 926
+    },
+    {
+      "epoch": 1.1537025513378967,
+      "grad_norm": 2.977605104446411,
+      "learning_rate": 3.4524907749077493e-06,
+      "loss": 1.4747,
+      "step": 927
+    },
+    {
+      "epoch": 1.1549471064094585,
+      "grad_norm": 2.9327852725982666,
+      "learning_rate": 3.4501845018450187e-06,
+      "loss": 1.4219,
+      "step": 928
+    },
+    {
+      "epoch": 1.1561916614810206,
+      "grad_norm": 2.844038963317871,
+      "learning_rate": 3.447878228782288e-06,
+      "loss": 1.5146,
+      "step": 929
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "grad_norm": 2.9402201175689697,
+      "learning_rate": 3.4455719557195576e-06,
+      "loss": 1.45,
+      "step": 930
+    },
+    {
+      "epoch": 1.1574362165525824,
+      "eval_loss": 1.6620122194290161,
+      "eval_runtime": 46.1748,
+      "eval_samples_per_second": 21.657,
+      "eval_steps_per_second": 0.91,
+      "step": 930
+    },
+    {
+      "epoch": 1.1586807716241443,
+      "grad_norm": 2.842651128768921,
+      "learning_rate": 3.4432656826568266e-06,
+      "loss": 1.4415,
+      "step": 931
+    },
+    {
+      "epoch": 1.1599253266957064,
+      "grad_norm": 2.8909685611724854,
+      "learning_rate": 3.4409594095940964e-06,
+      "loss": 1.4529,
+      "step": 932
+    },
+    {
+      "epoch": 1.1611698817672682,
+      "grad_norm": 2.8025808334350586,
+      "learning_rate": 3.438653136531366e-06,
+      "loss": 1.4552,
+      "step": 933
+    },
+    {
+      "epoch": 1.16241443683883,
+      "grad_norm": 2.6995794773101807,
+      "learning_rate": 3.436346863468635e-06,
+      "loss": 1.4435,
+      "step": 934
+    },
+    {
+      "epoch": 1.1636589919103921,
+      "grad_norm": 2.7307121753692627,
+      "learning_rate": 3.4340405904059043e-06,
+      "loss": 1.4447,
+      "step": 935
+    },
+    {
+      "epoch": 1.164903546981954,
+      "grad_norm": 2.9532501697540283,
+      "learning_rate": 3.4317343173431737e-06,
+      "loss": 1.4747,
+      "step": 936
+    },
+    {
+      "epoch": 1.1661481020535158,
+      "grad_norm": 3.0135788917541504,
+      "learning_rate": 3.4294280442804427e-06,
+      "loss": 1.525,
+      "step": 937
+    },
+    {
+      "epoch": 1.167392657125078,
+      "grad_norm": 2.796355962753296,
+      "learning_rate": 3.4271217712177125e-06,
+      "loss": 1.4539,
+      "step": 938
+    },
+    {
+      "epoch": 1.1686372121966397,
+      "grad_norm": 2.9422309398651123,
+      "learning_rate": 3.424815498154982e-06,
+      "loss": 1.4664,
+      "step": 939
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "grad_norm": 2.7680039405822754,
+      "learning_rate": 3.4225092250922514e-06,
+      "loss": 1.4837,
+      "step": 940
+    },
+    {
+      "epoch": 1.1698817672682016,
+      "eval_loss": 1.660908818244934,
+      "eval_runtime": 45.6998,
+      "eval_samples_per_second": 21.882,
+      "eval_steps_per_second": 0.919,
+      "step": 940
+    },
+    {
+      "epoch": 1.1711263223397634,
+      "grad_norm": 2.8888845443725586,
+      "learning_rate": 3.4202029520295204e-06,
+      "loss": 1.5315,
+      "step": 941
+    },
+    {
+      "epoch": 1.1723708774113255,
+      "grad_norm": 2.7124130725860596,
+      "learning_rate": 3.41789667896679e-06,
+      "loss": 1.4188,
+      "step": 942
+    },
+    {
+      "epoch": 1.1736154324828874,
+      "grad_norm": 2.75203013420105,
+      "learning_rate": 3.4155904059040596e-06,
+      "loss": 1.4533,
+      "step": 943
+    },
+    {
+      "epoch": 1.1748599875544492,
+      "grad_norm": 2.865480422973633,
+      "learning_rate": 3.4132841328413286e-06,
+      "loss": 1.4532,
+      "step": 944
+    },
+    {
+      "epoch": 1.1761045426260113,
+      "grad_norm": 2.8618321418762207,
+      "learning_rate": 3.410977859778598e-06,
+      "loss": 1.4386,
+      "step": 945
+    },
+    {
+      "epoch": 1.1773490976975731,
+      "grad_norm": 2.7656145095825195,
+      "learning_rate": 3.4086715867158675e-06,
+      "loss": 1.3996,
+      "step": 946
+    },
+    {
+      "epoch": 1.178593652769135,
+      "grad_norm": 2.76213002204895,
+      "learning_rate": 3.4063653136531365e-06,
+      "loss": 1.4979,
+      "step": 947
+    },
+    {
+      "epoch": 1.179838207840697,
+      "grad_norm": 2.8581957817077637,
+      "learning_rate": 3.4040590405904063e-06,
+      "loss": 1.4673,
+      "step": 948
+    },
+    {
+      "epoch": 1.1810827629122589,
+      "grad_norm": 2.7825586795806885,
+      "learning_rate": 3.4017527675276758e-06,
+      "loss": 1.4453,
+      "step": 949
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "grad_norm": 2.9226131439208984,
+      "learning_rate": 3.3994464944649448e-06,
+      "loss": 1.4879,
+      "step": 950
+    },
+    {
+      "epoch": 1.1823273179838207,
+      "eval_loss": 1.6621263027191162,
+      "eval_runtime": 45.1222,
+      "eval_samples_per_second": 22.162,
+      "eval_steps_per_second": 0.931,
+      "step": 950
+    },
+    {
+      "epoch": 1.1835718730553828,
+      "grad_norm": 2.9997975826263428,
+      "learning_rate": 3.397140221402214e-06,
+      "loss": 1.505,
+      "step": 951
+    },
+    {
+      "epoch": 1.1848164281269447,
+      "grad_norm": 2.7225453853607178,
+      "learning_rate": 3.3948339483394836e-06,
+      "loss": 1.4695,
+      "step": 952
+    },
+    {
+      "epoch": 1.1860609831985065,
+      "grad_norm": 2.916473150253296,
+      "learning_rate": 3.3925276752767534e-06,
+      "loss": 1.4901,
+      "step": 953
+    },
+    {
+      "epoch": 1.1873055382700684,
+      "grad_norm": 2.8371644020080566,
+      "learning_rate": 3.3902214022140224e-06,
+      "loss": 1.4944,
+      "step": 954
+    },
+    {
+      "epoch": 1.1885500933416304,
+      "grad_norm": 2.813662528991699,
+      "learning_rate": 3.387915129151292e-06,
+      "loss": 1.4603,
+      "step": 955
+    },
+    {
+      "epoch": 1.1897946484131923,
+      "grad_norm": 2.991931676864624,
+      "learning_rate": 3.3856088560885613e-06,
+      "loss": 1.5496,
+      "step": 956
+    },
+    {
+      "epoch": 1.1910392034847541,
+      "grad_norm": 2.6252501010894775,
+      "learning_rate": 3.3833025830258303e-06,
+      "loss": 1.4405,
+      "step": 957
+    },
+    {
+      "epoch": 1.1922837585563162,
+      "grad_norm": 2.747725009918213,
+      "learning_rate": 3.3809963099630997e-06,
+      "loss": 1.4689,
+      "step": 958
+    },
+    {
+      "epoch": 1.193528313627878,
+      "grad_norm": 2.887763261795044,
+      "learning_rate": 3.3786900369003696e-06,
+      "loss": 1.4501,
+      "step": 959
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "grad_norm": 2.9954354763031006,
+      "learning_rate": 3.3763837638376386e-06,
+      "loss": 1.4978,
+      "step": 960
+    },
+    {
+      "epoch": 1.1947728686994399,
+      "eval_loss": 1.664442539215088,
+      "eval_runtime": 49.1266,
+      "eval_samples_per_second": 20.356,
+      "eval_steps_per_second": 0.855,
+      "step": 960
+    },
+    {
+      "epoch": 1.196017423771002,
+      "grad_norm": 3.0174152851104736,
+      "learning_rate": 3.374077490774908e-06,
+      "loss": 1.4353,
+      "step": 961
+    },
+    {
+      "epoch": 1.1972619788425638,
+      "grad_norm": 2.9082882404327393,
+      "learning_rate": 3.3717712177121774e-06,
+      "loss": 1.4878,
+      "step": 962
+    },
+    {
+      "epoch": 1.1985065339141256,
+      "grad_norm": 2.897993326187134,
+      "learning_rate": 3.3694649446494464e-06,
+      "loss": 1.4975,
+      "step": 963
+    },
+    {
+      "epoch": 1.1997510889856877,
+      "grad_norm": 2.739470958709717,
+      "learning_rate": 3.3671586715867163e-06,
+      "loss": 1.4202,
+      "step": 964
+    },
+    {
+      "epoch": 1.2009956440572496,
+      "grad_norm": 2.817786693572998,
+      "learning_rate": 3.3648523985239857e-06,
+      "loss": 1.5195,
+      "step": 965
+    },
+    {
+      "epoch": 1.2022401991288114,
+      "grad_norm": 2.8476386070251465,
+      "learning_rate": 3.362546125461255e-06,
+      "loss": 1.444,
+      "step": 966
+    },
+    {
+      "epoch": 1.2034847542003733,
+      "grad_norm": 2.804047107696533,
+      "learning_rate": 3.360239852398524e-06,
+      "loss": 1.4442,
+      "step": 967
+    },
+    {
+      "epoch": 1.2047293092719353,
+      "grad_norm": 2.9285430908203125,
+      "learning_rate": 3.3579335793357935e-06,
+      "loss": 1.4877,
+      "step": 968
+    },
+    {
+      "epoch": 1.2059738643434972,
+      "grad_norm": 2.8814878463745117,
+      "learning_rate": 3.3556273062730634e-06,
+      "loss": 1.5078,
+      "step": 969
+    },
+    {
+      "epoch": 1.207218419415059,
+      "grad_norm": 2.970728635787964,
+      "learning_rate": 3.3533210332103324e-06,
+      "loss": 1.4166,
+      "step": 970
+    },
+    {
+      "epoch": 1.207218419415059,
+      "eval_loss": 1.6635315418243408,
+      "eval_runtime": 47.2321,
+      "eval_samples_per_second": 21.172,
+      "eval_steps_per_second": 0.889,
+      "step": 970
+    },
+    {
+      "epoch": 1.208462974486621,
+      "grad_norm": 2.9836292266845703,
+      "learning_rate": 3.3510147601476018e-06,
+      "loss": 1.4777,
+      "step": 971
+    },
+    {
+      "epoch": 1.209707529558183,
+      "grad_norm": 2.8495216369628906,
+      "learning_rate": 3.348708487084871e-06,
+      "loss": 1.4829,
+      "step": 972
+    },
+    {
+      "epoch": 1.2109520846297448,
+      "grad_norm": 2.7709312438964844,
+      "learning_rate": 3.34640221402214e-06,
+      "loss": 1.4242,
+      "step": 973
+    },
+    {
+      "epoch": 1.2121966397013069,
+      "grad_norm": 2.999462366104126,
+      "learning_rate": 3.3440959409594096e-06,
+      "loss": 1.4148,
+      "step": 974
+    },
+    {
+      "epoch": 1.2134411947728687,
+      "grad_norm": 2.8719074726104736,
+      "learning_rate": 3.3417896678966795e-06,
+      "loss": 1.4196,
+      "step": 975
+    },
+    {
+      "epoch": 1.2146857498444306,
+      "grad_norm": 2.8469338417053223,
+      "learning_rate": 3.3394833948339485e-06,
+      "loss": 1.4941,
+      "step": 976
+    },
+    {
+      "epoch": 1.2159303049159926,
+      "grad_norm": 2.9780938625335693,
+      "learning_rate": 3.337177121771218e-06,
+      "loss": 1.513,
+      "step": 977
+    },
+    {
+      "epoch": 1.2171748599875545,
+      "grad_norm": 2.814760684967041,
+      "learning_rate": 3.3348708487084873e-06,
+      "loss": 1.4204,
+      "step": 978
+    },
+    {
+      "epoch": 1.2184194150591163,
+      "grad_norm": 3.016261339187622,
+      "learning_rate": 3.332564575645757e-06,
+      "loss": 1.4389,
+      "step": 979
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "grad_norm": 2.9385855197906494,
+      "learning_rate": 3.330258302583026e-06,
+      "loss": 1.4885,
+      "step": 980
+    },
+    {
+      "epoch": 1.2196639701306782,
+      "eval_loss": 1.6688873767852783,
+      "eval_runtime": 46.2942,
+      "eval_samples_per_second": 21.601,
+      "eval_steps_per_second": 0.907,
+      "step": 980
+    },
+    {
+      "epoch": 1.2209085252022402,
+      "grad_norm": 3.1434757709503174,
+      "learning_rate": 3.3279520295202956e-06,
+      "loss": 1.5157,
+      "step": 981
+    },
+    {
+      "epoch": 1.222153080273802,
+      "grad_norm": 3.0104827880859375,
+      "learning_rate": 3.325645756457565e-06,
+      "loss": 1.49,
+      "step": 982
+    },
+    {
+      "epoch": 1.223397635345364,
+      "grad_norm": 3.1913645267486572,
+      "learning_rate": 3.323339483394834e-06,
+      "loss": 1.4937,
+      "step": 983
+    },
+    {
+      "epoch": 1.224642190416926,
+      "grad_norm": 3.19496750831604,
+      "learning_rate": 3.3210332103321034e-06,
+      "loss": 1.5197,
+      "step": 984
+    },
+    {
+      "epoch": 1.2258867454884879,
+      "grad_norm": 3.046232223510742,
+      "learning_rate": 3.3187269372693733e-06,
+      "loss": 1.4497,
+      "step": 985
+    },
+    {
+      "epoch": 1.2271313005600497,
+      "grad_norm": 2.850675344467163,
+      "learning_rate": 3.3164206642066423e-06,
+      "loss": 1.4595,
+      "step": 986
+    },
+    {
+      "epoch": 1.2283758556316118,
+      "grad_norm": 2.7744576930999756,
+      "learning_rate": 3.3141143911439117e-06,
+      "loss": 1.4726,
+      "step": 987
+    },
+    {
+      "epoch": 1.2296204107031736,
+      "grad_norm": 3.017153024673462,
+      "learning_rate": 3.311808118081181e-06,
+      "loss": 1.493,
+      "step": 988
+    },
+    {
+      "epoch": 1.2308649657747355,
+      "grad_norm": 2.8821120262145996,
+      "learning_rate": 3.3095018450184506e-06,
+      "loss": 1.4734,
+      "step": 989
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "grad_norm": 2.8805840015411377,
+      "learning_rate": 3.3071955719557196e-06,
+      "loss": 1.4562,
+      "step": 990
+    },
+    {
+      "epoch": 1.2321095208462975,
+      "eval_loss": 1.669345736503601,
+      "eval_runtime": 47.0893,
+      "eval_samples_per_second": 21.236,
+      "eval_steps_per_second": 0.892,
+      "step": 990
+    },
+    {
+      "epoch": 1.2333540759178594,
+      "grad_norm": 2.7599008083343506,
+      "learning_rate": 3.3048892988929894e-06,
+      "loss": 1.4218,
+      "step": 991
+    },
+    {
+      "epoch": 1.2345986309894212,
+      "grad_norm": 2.9835240840911865,
+      "learning_rate": 3.302583025830259e-06,
+      "loss": 1.4596,
+      "step": 992
+    },
+    {
+      "epoch": 1.235843186060983,
+      "grad_norm": 2.843358039855957,
+      "learning_rate": 3.300276752767528e-06,
+      "loss": 1.4561,
+      "step": 993
+    },
+    {
+      "epoch": 1.2370877411325452,
+      "grad_norm": 2.7386093139648438,
+      "learning_rate": 3.2979704797047972e-06,
+      "loss": 1.4475,
+      "step": 994
+    },
+    {
+      "epoch": 1.238332296204107,
+      "grad_norm": 2.668161392211914,
+      "learning_rate": 3.2956642066420667e-06,
+      "loss": 1.4661,
+      "step": 995
+    },
+    {
+      "epoch": 1.2395768512756689,
+      "grad_norm": 2.947235107421875,
+      "learning_rate": 3.2933579335793357e-06,
+      "loss": 1.4358,
+      "step": 996
+    },
+    {
+      "epoch": 1.240821406347231,
+      "grad_norm": 2.8159878253936768,
+      "learning_rate": 3.2910516605166055e-06,
+      "loss": 1.4432,
+      "step": 997
+    },
+    {
+      "epoch": 1.2420659614187928,
+      "grad_norm": 2.896597385406494,
+      "learning_rate": 3.288745387453875e-06,
+      "loss": 1.5585,
+      "step": 998
+    },
+    {
+      "epoch": 1.2433105164903546,
+      "grad_norm": 2.80241322517395,
+      "learning_rate": 3.286439114391144e-06,
+      "loss": 1.5076,
+      "step": 999
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "grad_norm": 3.008939027786255,
+      "learning_rate": 3.2841328413284134e-06,
+      "loss": 1.4234,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2445550715619167,
+      "eval_loss": 1.6621588468551636,
+      "eval_runtime": 50.4926,
+      "eval_samples_per_second": 19.805,
+      "eval_steps_per_second": 0.832,
+      "step": 1000
+    },
+    {
+      "epoch": 1.2457996266334785,
+      "grad_norm": 2.9872381687164307,
+      "learning_rate": 3.281826568265683e-06,
+      "loss": 1.453,
+      "step": 1001
+    },
+    {
+      "epoch": 1.2470441817050404,
+      "grad_norm": 2.704676628112793,
+      "learning_rate": 3.2795202952029526e-06,
+      "loss": 1.4711,
+      "step": 1002
+    },
+    {
+      "epoch": 1.2482887367766025,
+      "grad_norm": 2.8100757598876953,
+      "learning_rate": 3.2772140221402216e-06,
+      "loss": 1.3884,
+      "step": 1003
+    },
+    {
+      "epoch": 1.2495332918481643,
+      "grad_norm": 2.902953624725342,
+      "learning_rate": 3.274907749077491e-06,
+      "loss": 1.4986,
+      "step": 1004
+    },
+    {
+      "epoch": 1.2507778469197262,
+      "grad_norm": 2.980163097381592,
+      "learning_rate": 3.2726014760147605e-06,
+      "loss": 1.5067,
+      "step": 1005
+    },
+    {
+      "epoch": 1.252022401991288,
+      "grad_norm": 2.8820345401763916,
+      "learning_rate": 3.2702952029520295e-06,
+      "loss": 1.4995,
+      "step": 1006
+    },
+    {
+      "epoch": 1.25326695706285,
+      "grad_norm": 2.8219635486602783,
+      "learning_rate": 3.2679889298892993e-06,
+      "loss": 1.4881,
+      "step": 1007
+    },
+    {
+      "epoch": 1.254511512134412,
+      "grad_norm": 2.8593199253082275,
+      "learning_rate": 3.2656826568265687e-06,
+      "loss": 1.4469,
+      "step": 1008
+    },
+    {
+      "epoch": 1.255756067205974,
+      "grad_norm": 2.888357639312744,
+      "learning_rate": 3.2633763837638377e-06,
+      "loss": 1.4869,
+      "step": 1009
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "grad_norm": 2.9985013008117676,
+      "learning_rate": 3.261070110701107e-06,
+      "loss": 1.4724,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2570006222775358,
+      "eval_loss": 1.6492141485214233,
+      "eval_runtime": 45.2197,
+      "eval_samples_per_second": 22.114,
+      "eval_steps_per_second": 0.929,
+      "step": 1010
+    },
+    {
+      "epoch": 1.2582451773490977,
+      "grad_norm": 2.860780715942383,
+      "learning_rate": 3.2587638376383766e-06,
+      "loss": 1.4618,
+      "step": 1011
+    },
+    {
+      "epoch": 1.2594897324206595,
+      "grad_norm": 2.766479730606079,
+      "learning_rate": 3.2564575645756456e-06,
+      "loss": 1.4642,
+      "step": 1012
+    },
+    {
+      "epoch": 1.2607342874922216,
+      "grad_norm": 2.711759328842163,
+      "learning_rate": 3.2541512915129154e-06,
+      "loss": 1.4283,
+      "step": 1013
+    },
+    {
+      "epoch": 1.2619788425637835,
+      "grad_norm": 2.8365769386291504,
+      "learning_rate": 3.251845018450185e-06,
+      "loss": 1.4248,
+      "step": 1014
+    },
+    {
+      "epoch": 1.2632233976353453,
+      "grad_norm": 2.957620143890381,
+      "learning_rate": 3.2495387453874543e-06,
+      "loss": 1.4662,
+      "step": 1015
+    },
+    {
+      "epoch": 1.2644679527069074,
+      "grad_norm": 2.8066208362579346,
+      "learning_rate": 3.2472324723247233e-06,
+      "loss": 1.4447,
+      "step": 1016
+    },
+    {
+      "epoch": 1.2657125077784692,
+      "grad_norm": 2.902040958404541,
+      "learning_rate": 3.244926199261993e-06,
+      "loss": 1.4743,
+      "step": 1017
+    },
+    {
+      "epoch": 1.266957062850031,
+      "grad_norm": 2.7636687755584717,
+      "learning_rate": 3.2426199261992625e-06,
+      "loss": 1.4465,
+      "step": 1018
+    },
+    {
+      "epoch": 1.268201617921593,
+      "grad_norm": 2.95261549949646,
+      "learning_rate": 3.2403136531365315e-06,
+      "loss": 1.4769,
+      "step": 1019
+    },
+    {
+      "epoch": 1.269446172993155,
+      "grad_norm": 2.911128520965576,
+      "learning_rate": 3.238007380073801e-06,
+      "loss": 1.3843,
+      "step": 1020
+    },
+    {
+      "epoch": 1.269446172993155,
+      "eval_loss": 1.6526964902877808,
+      "eval_runtime": 48.2126,
+      "eval_samples_per_second": 20.741,
+      "eval_steps_per_second": 0.871,
+      "step": 1020
+    },
+    {
+      "epoch": 1.2706907280647168,
+      "grad_norm": 2.769951820373535,
+      "learning_rate": 3.2357011070110704e-06,
+      "loss": 1.4847,
+      "step": 1021
+    },
+    {
+      "epoch": 1.271935283136279,
+      "grad_norm": 3.0451674461364746,
+      "learning_rate": 3.2333948339483394e-06,
+      "loss": 1.4453,
+      "step": 1022
+    },
+    {
+      "epoch": 1.2731798382078408,
+      "grad_norm": 2.9543521404266357,
+      "learning_rate": 3.2310885608856092e-06,
+      "loss": 1.4789,
+      "step": 1023
+    },
+    {
+      "epoch": 1.2744243932794026,
+      "grad_norm": 2.9174394607543945,
+      "learning_rate": 3.2287822878228787e-06,
+      "loss": 1.4807,
+      "step": 1024
+    },
+    {
+      "epoch": 1.2756689483509644,
+      "grad_norm": 2.888258934020996,
+      "learning_rate": 3.2264760147601477e-06,
+      "loss": 1.4367,
+      "step": 1025
+    },
+    {
+      "epoch": 1.2769135034225265,
+      "grad_norm": 2.8346047401428223,
+      "learning_rate": 3.224169741697417e-06,
+      "loss": 1.4313,
+      "step": 1026
+    },
+    {
+      "epoch": 1.2781580584940884,
+      "grad_norm": 2.7531332969665527,
+      "learning_rate": 3.2218634686346865e-06,
+      "loss": 1.4018,
+      "step": 1027
+    },
+    {
+      "epoch": 1.2794026135656502,
+      "grad_norm": 3.136946439743042,
+      "learning_rate": 3.2195571955719564e-06,
+      "loss": 1.4417,
+      "step": 1028
+    },
+    {
+      "epoch": 1.2806471686372123,
+      "grad_norm": 2.9618165493011475,
+      "learning_rate": 3.2172509225092254e-06,
+      "loss": 1.5117,
+      "step": 1029
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "grad_norm": 3.0740747451782227,
+      "learning_rate": 3.2149446494464948e-06,
+      "loss": 1.4641,
+      "step": 1030
+    },
+    {
+      "epoch": 1.2818917237087741,
+      "eval_loss": 1.6572761535644531,
+      "eval_runtime": 51.6696,
+      "eval_samples_per_second": 19.354,
+      "eval_steps_per_second": 0.813,
+      "step": 1030
+    },
+    {
+      "epoch": 1.283136278780336,
+      "grad_norm": 2.9225106239318848,
+      "learning_rate": 3.212638376383764e-06,
+      "loss": 1.4421,
+      "step": 1031
+    },
+    {
+      "epoch": 1.2843808338518978,
+      "grad_norm": 2.9530649185180664,
+      "learning_rate": 3.210332103321033e-06,
+      "loss": 1.5138,
+      "step": 1032
+    },
+    {
+      "epoch": 1.28562538892346,
+      "grad_norm": 2.8513238430023193,
+      "learning_rate": 3.208025830258303e-06,
+      "loss": 1.411,
+      "step": 1033
+    },
+    {
+      "epoch": 1.2868699439950217,
+      "grad_norm": 2.7440171241760254,
+      "learning_rate": 3.2057195571955725e-06,
+      "loss": 1.4182,
+      "step": 1034
+    },
+    {
+      "epoch": 1.2881144990665838,
+      "grad_norm": 2.8126630783081055,
+      "learning_rate": 3.2034132841328415e-06,
+      "loss": 1.4333,
+      "step": 1035
+    },
+    {
+      "epoch": 1.2893590541381457,
+      "grad_norm": 2.8598744869232178,
+      "learning_rate": 3.201107011070111e-06,
+      "loss": 1.439,
+      "step": 1036
+    },
+    {
+      "epoch": 1.2906036092097075,
+      "grad_norm": 2.901622772216797,
+      "learning_rate": 3.1988007380073803e-06,
+      "loss": 1.4461,
+      "step": 1037
+    },
+    {
+      "epoch": 1.2918481642812694,
+      "grad_norm": 2.7670180797576904,
+      "learning_rate": 3.1964944649446493e-06,
+      "loss": 1.4772,
+      "step": 1038
+    },
+    {
+      "epoch": 1.2930927193528314,
+      "grad_norm": 2.7860350608825684,
+      "learning_rate": 3.194188191881919e-06,
+      "loss": 1.4414,
+      "step": 1039
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "grad_norm": 2.822052478790283,
+      "learning_rate": 3.1918819188191886e-06,
+      "loss": 1.4192,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2943372744243933,
+      "eval_loss": 1.663284420967102,
+      "eval_runtime": 47.2822,
+      "eval_samples_per_second": 21.15,
+      "eval_steps_per_second": 0.888,
+      "step": 1040
+    },
+    {
+      "epoch": 1.2955818294959551,
+      "grad_norm": 2.909109115600586,
+      "learning_rate": 3.189575645756458e-06,
+      "loss": 1.4795,
+      "step": 1041
+    },
+    {
+      "epoch": 1.2968263845675172,
+      "grad_norm": 2.933673143386841,
+      "learning_rate": 3.187269372693727e-06,
+      "loss": 1.5038,
+      "step": 1042
+    },
+    {
+      "epoch": 1.298070939639079,
+      "grad_norm": 3.032518148422241,
+      "learning_rate": 3.1849630996309964e-06,
+      "loss": 1.5133,
+      "step": 1043
+    },
+    {
+      "epoch": 1.299315494710641,
+      "grad_norm": 2.826040506362915,
+      "learning_rate": 3.1826568265682663e-06,
+      "loss": 1.4395,
+      "step": 1044
+    },
+    {
+      "epoch": 1.3005600497822027,
+      "grad_norm": 2.8152658939361572,
+      "learning_rate": 3.1803505535055353e-06,
+      "loss": 1.4787,
+      "step": 1045
+    },
+    {
+      "epoch": 1.3018046048537648,
+      "grad_norm": 2.9410431385040283,
+      "learning_rate": 3.1780442804428047e-06,
+      "loss": 1.4375,
+      "step": 1046
+    },
+    {
+      "epoch": 1.3030491599253267,
+      "grad_norm": 2.948636531829834,
+      "learning_rate": 3.175738007380074e-06,
+      "loss": 1.4847,
+      "step": 1047
+    },
+    {
+      "epoch": 1.3042937149968887,
+      "grad_norm": 2.6287615299224854,
+      "learning_rate": 3.173431734317343e-06,
+      "loss": 1.4186,
+      "step": 1048
+    },
+    {
+      "epoch": 1.3055382700684506,
+      "grad_norm": 2.836029052734375,
+      "learning_rate": 3.1711254612546125e-06,
+      "loss": 1.4251,
+      "step": 1049
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "grad_norm": 2.982081651687622,
+      "learning_rate": 3.1688191881918824e-06,
+      "loss": 1.4951,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3067828251400124,
+      "eval_loss": 1.6578067541122437,
+      "eval_runtime": 47.5847,
+      "eval_samples_per_second": 21.015,
+      "eval_steps_per_second": 0.883,
+      "step": 1050
+    },
+    {
+      "epoch": 1.3080273802115743,
+      "grad_norm": 2.9297118186950684,
+      "learning_rate": 3.166512915129152e-06,
+      "loss": 1.4699,
+      "step": 1051
+    },
+    {
+      "epoch": 1.3092719352831363,
+      "grad_norm": 2.906435251235962,
+      "learning_rate": 3.164206642066421e-06,
+      "loss": 1.4562,
+      "step": 1052
+    },
+    {
+      "epoch": 1.3105164903546982,
+      "grad_norm": 2.8655896186828613,
+      "learning_rate": 3.1619003690036902e-06,
+      "loss": 1.4351,
+      "step": 1053
+    },
+    {
+      "epoch": 1.31176104542626,
+      "grad_norm": 2.779244899749756,
+      "learning_rate": 3.15959409594096e-06,
+      "loss": 1.4479,
+      "step": 1054
+    },
+    {
+      "epoch": 1.3130056004978221,
+      "grad_norm": 2.9863739013671875,
+      "learning_rate": 3.157287822878229e-06,
+      "loss": 1.4964,
+      "step": 1055
+    },
+    {
+      "epoch": 1.314250155569384,
+      "grad_norm": 2.821871042251587,
+      "learning_rate": 3.1549815498154985e-06,
+      "loss": 1.4173,
+      "step": 1056
+    },
+    {
+      "epoch": 1.3154947106409458,
+      "grad_norm": 2.8573882579803467,
+      "learning_rate": 3.152675276752768e-06,
+      "loss": 1.4877,
+      "step": 1057
+    },
+    {
+      "epoch": 1.3167392657125077,
+      "grad_norm": 2.9824137687683105,
+      "learning_rate": 3.150369003690037e-06,
+      "loss": 1.5124,
+      "step": 1058
+    },
+    {
+      "epoch": 1.3179838207840697,
+      "grad_norm": 2.881098747253418,
+      "learning_rate": 3.1480627306273063e-06,
+      "loss": 1.4346,
+      "step": 1059
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "grad_norm": 2.918782949447632,
+      "learning_rate": 3.145756457564576e-06,
+      "loss": 1.4797,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3192283758556316,
+      "eval_loss": 1.6510179042816162,
+      "eval_runtime": 53.6356,
+      "eval_samples_per_second": 18.644,
+      "eval_steps_per_second": 0.783,
+      "step": 1060
+    },
+    {
+      "epoch": 1.3204729309271936,
+      "grad_norm": 3.045776128768921,
+      "learning_rate": 3.143450184501845e-06,
+      "loss": 1.5039,
+      "step": 1061
+    },
+    {
+      "epoch": 1.3217174859987555,
+      "grad_norm": 3.041501760482788,
+      "learning_rate": 3.1411439114391146e-06,
+      "loss": 1.4878,
+      "step": 1062
+    },
+    {
+      "epoch": 1.3229620410703173,
+      "grad_norm": 2.917837142944336,
+      "learning_rate": 3.138837638376384e-06,
+      "loss": 1.4202,
+      "step": 1063
+    },
+    {
+      "epoch": 1.3242065961418792,
+      "grad_norm": 2.708070755004883,
+      "learning_rate": 3.1365313653136535e-06,
+      "loss": 1.4071,
+      "step": 1064
+    },
+    {
+      "epoch": 1.3254511512134413,
+      "grad_norm": 2.9146156311035156,
+      "learning_rate": 3.1342250922509225e-06,
+      "loss": 1.4428,
+      "step": 1065
+    },
+    {
+      "epoch": 1.326695706285003,
+      "grad_norm": 3.220919132232666,
+      "learning_rate": 3.1319188191881923e-06,
+      "loss": 1.4733,
+      "step": 1066
+    },
+    {
+      "epoch": 1.327940261356565,
+      "grad_norm": 2.9127554893493652,
+      "learning_rate": 3.1296125461254617e-06,
+      "loss": 1.4554,
+      "step": 1067
+    },
+    {
+      "epoch": 1.329184816428127,
+      "grad_norm": 3.053191661834717,
+      "learning_rate": 3.1273062730627307e-06,
+      "loss": 1.4044,
+      "step": 1068
+    },
+    {
+      "epoch": 1.3304293714996889,
+      "grad_norm": 3.2937021255493164,
+      "learning_rate": 3.125e-06,
+      "loss": 1.4949,
+      "step": 1069
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "grad_norm": 2.976717948913574,
+      "learning_rate": 3.12269372693727e-06,
+      "loss": 1.446,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3316739265712507,
+      "eval_loss": 1.6503105163574219,
+      "eval_runtime": 50.7579,
+      "eval_samples_per_second": 19.701,
+      "eval_steps_per_second": 0.827,
+      "step": 1070
+    },
+    {
+      "epoch": 1.3329184816428126,
+      "grad_norm": 3.0860936641693115,
+      "learning_rate": 3.120387453874539e-06,
+      "loss": 1.4055,
+      "step": 1071
+    },
+    {
+      "epoch": 1.3341630367143746,
+      "grad_norm": 3.0401217937469482,
+      "learning_rate": 3.1180811808118084e-06,
+      "loss": 1.4688,
+      "step": 1072
+    },
+    {
+      "epoch": 1.3354075917859365,
+      "grad_norm": 2.953253984451294,
+      "learning_rate": 3.115774907749078e-06,
+      "loss": 1.5001,
+      "step": 1073
+    },
+    {
+      "epoch": 1.3366521468574986,
+      "grad_norm": 3.0665059089660645,
+      "learning_rate": 3.113468634686347e-06,
+      "loss": 1.4217,
+      "step": 1074
+    },
+    {
+      "epoch": 1.3378967019290604,
+      "grad_norm": 3.121600389480591,
+      "learning_rate": 3.1111623616236163e-06,
+      "loss": 1.4508,
+      "step": 1075
+    },
+    {
+      "epoch": 1.3391412570006223,
+      "grad_norm": 2.8057339191436768,
+      "learning_rate": 3.108856088560886e-06,
+      "loss": 1.4241,
+      "step": 1076
+    },
+    {
+      "epoch": 1.340385812072184,
+      "grad_norm": 3.0147411823272705,
+      "learning_rate": 3.1065498154981555e-06,
+      "loss": 1.479,
+      "step": 1077
+    },
+    {
+      "epoch": 1.3416303671437462,
+      "grad_norm": 3.030010223388672,
+      "learning_rate": 3.1042435424354245e-06,
+      "loss": 1.4861,
+      "step": 1078
+    },
+    {
+      "epoch": 1.342874922215308,
+      "grad_norm": 3.027998208999634,
+      "learning_rate": 3.101937269372694e-06,
+      "loss": 1.4916,
+      "step": 1079
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "grad_norm": 2.9252536296844482,
+      "learning_rate": 3.0996309963099634e-06,
+      "loss": 1.418,
+      "step": 1080
+    },
+    {
+      "epoch": 1.3441194772868699,
+      "eval_loss": 1.6490333080291748,
+      "eval_runtime": 47.4026,
+      "eval_samples_per_second": 21.096,
+      "eval_steps_per_second": 0.886,
+      "step": 1080
+    },
+    {
+      "epoch": 1.345364032358432,
+      "grad_norm": 3.110847234725952,
+      "learning_rate": 3.0973247232472324e-06,
+      "loss": 1.4127,
+      "step": 1081
+    },
+    {
+      "epoch": 1.3466085874299938,
+      "grad_norm": 2.846491575241089,
+      "learning_rate": 3.0950184501845022e-06,
+      "loss": 1.444,
+      "step": 1082
+    },
+    {
+      "epoch": 1.3478531425015556,
+      "grad_norm": 2.950533151626587,
+      "learning_rate": 3.0927121771217716e-06,
+      "loss": 1.4456,
+      "step": 1083
+    },
+    {
+      "epoch": 1.3490976975731175,
+      "grad_norm": 2.875964403152466,
+      "learning_rate": 3.0904059040590406e-06,
+      "loss": 1.4485,
+      "step": 1084
+    },
+    {
+      "epoch": 1.3503422526446796,
+      "grad_norm": 2.9612326622009277,
+      "learning_rate": 3.08809963099631e-06,
+      "loss": 1.4446,
+      "step": 1085
+    },
+    {
+      "epoch": 1.3515868077162414,
+      "grad_norm": 2.925297975540161,
+      "learning_rate": 3.0857933579335795e-06,
+      "loss": 1.4847,
+      "step": 1086
+    },
+    {
+      "epoch": 1.3528313627878035,
+      "grad_norm": 2.991616725921631,
+      "learning_rate": 3.0834870848708485e-06,
+      "loss": 1.5151,
+      "step": 1087
+    },
+    {
+      "epoch": 1.3540759178593653,
+      "grad_norm": 2.96329927444458,
+      "learning_rate": 3.0811808118081183e-06,
+      "loss": 1.4574,
+      "step": 1088
+    },
+    {
+      "epoch": 1.3553204729309272,
+      "grad_norm": 2.948702573776245,
+      "learning_rate": 3.0788745387453878e-06,
+      "loss": 1.4585,
+      "step": 1089
+    },
+    {
+      "epoch": 1.356565028002489,
+      "grad_norm": 3.218463659286499,
+      "learning_rate": 3.076568265682657e-06,
+      "loss": 1.4439,
+      "step": 1090
+    },
+    {
+      "epoch": 1.356565028002489,
+      "eval_loss": 1.645838975906372,
+      "eval_runtime": 47.048,
+      "eval_samples_per_second": 21.255,
+      "eval_steps_per_second": 0.893,
+      "step": 1090
+    },
+    {
+      "epoch": 1.357809583074051,
+      "grad_norm": 2.9799578189849854,
+      "learning_rate": 3.074261992619926e-06,
+      "loss": 1.4883,
+      "step": 1091
+    },
+    {
+      "epoch": 1.359054138145613,
+      "grad_norm": 2.7660505771636963,
+      "learning_rate": 3.071955719557196e-06,
+      "loss": 1.4349,
+      "step": 1092
+    },
+    {
+      "epoch": 1.3602986932171748,
+      "grad_norm": 3.0201916694641113,
+      "learning_rate": 3.0696494464944655e-06,
+      "loss": 1.418,
+      "step": 1093
+    },
+    {
+      "epoch": 1.3615432482887369,
+      "grad_norm": 2.844207763671875,
+      "learning_rate": 3.0673431734317345e-06,
+      "loss": 1.4769,
+      "step": 1094
+    },
+    {
+      "epoch": 1.3627878033602987,
+      "grad_norm": 2.8306753635406494,
+      "learning_rate": 3.065036900369004e-06,
+      "loss": 1.4653,
+      "step": 1095
+    },
+    {
+      "epoch": 1.3640323584318605,
+      "grad_norm": 2.805023431777954,
+      "learning_rate": 3.0627306273062733e-06,
+      "loss": 1.4771,
+      "step": 1096
+    },
+    {
+      "epoch": 1.3652769135034224,
+      "grad_norm": 3.020883083343506,
+      "learning_rate": 3.0604243542435423e-06,
+      "loss": 1.4237,
+      "step": 1097
+    },
+    {
+      "epoch": 1.3665214685749845,
+      "grad_norm": 2.8395731449127197,
+      "learning_rate": 3.058118081180812e-06,
+      "loss": 1.4647,
+      "step": 1098
+    },
+    {
+      "epoch": 1.3677660236465463,
+      "grad_norm": 2.871156692504883,
+      "learning_rate": 3.0558118081180816e-06,
+      "loss": 1.4196,
+      "step": 1099
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "grad_norm": 2.974905014038086,
+      "learning_rate": 3.053505535055351e-06,
+      "loss": 1.4779,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3690105787181084,
+      "eval_loss": 1.651503562927246,
+      "eval_runtime": 54.2713,
+      "eval_samples_per_second": 18.426,
+      "eval_steps_per_second": 0.774,
+      "step": 1100
+    },
+    {
+      "epoch": 1.3702551337896702,
+      "grad_norm": 2.7591328620910645,
+      "learning_rate": 3.05119926199262e-06,
+      "loss": 1.3899,
+      "step": 1101
+    },
+    {
+      "epoch": 1.371499688861232,
+      "grad_norm": 2.8575668334960938,
+      "learning_rate": 3.0488929889298894e-06,
+      "loss": 1.4482,
+      "step": 1102
+    },
+    {
+      "epoch": 1.372744243932794,
+      "grad_norm": 2.9049339294433594,
+      "learning_rate": 3.0465867158671593e-06,
+      "loss": 1.4785,
+      "step": 1103
+    },
+    {
+      "epoch": 1.373988799004356,
+      "grad_norm": 2.816453695297241,
+      "learning_rate": 3.0442804428044283e-06,
+      "loss": 1.4601,
+      "step": 1104
+    },
+    {
+      "epoch": 1.3752333540759178,
+      "grad_norm": 2.8844189643859863,
+      "learning_rate": 3.0419741697416977e-06,
+      "loss": 1.496,
+      "step": 1105
+    },
+    {
+      "epoch": 1.3764779091474797,
+      "grad_norm": 3.014744281768799,
+      "learning_rate": 3.039667896678967e-06,
+      "loss": 1.4098,
+      "step": 1106
+    },
+    {
+      "epoch": 1.3777224642190418,
+      "grad_norm": 2.8243560791015625,
+      "learning_rate": 3.037361623616236e-06,
+      "loss": 1.4086,
+      "step": 1107
+    },
+    {
+      "epoch": 1.3789670192906036,
+      "grad_norm": 2.8333253860473633,
+      "learning_rate": 3.035055350553506e-06,
+      "loss": 1.3516,
+      "step": 1108
+    },
+    {
+      "epoch": 1.3802115743621655,
+      "grad_norm": 2.926455020904541,
+      "learning_rate": 3.0327490774907754e-06,
+      "loss": 1.3976,
+      "step": 1109
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "grad_norm": 2.900937080383301,
+      "learning_rate": 3.0304428044280444e-06,
+      "loss": 1.4336,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3814561294337273,
+      "eval_loss": 1.647048830986023,
+      "eval_runtime": 42.3955,
+      "eval_samples_per_second": 23.587,
+      "eval_steps_per_second": 0.991,
+      "step": 1110
+    },
+    {
+      "epoch": 1.3827006845052894,
+      "grad_norm": 3.0756969451904297,
+      "learning_rate": 3.028136531365314e-06,
+      "loss": 1.4674,
+      "step": 1111
+    },
+    {
+      "epoch": 1.3839452395768512,
+      "grad_norm": 2.7754578590393066,
+      "learning_rate": 3.0258302583025832e-06,
+      "loss": 1.4435,
+      "step": 1112
+    },
+    {
+      "epoch": 1.3851897946484133,
+      "grad_norm": 2.8011393547058105,
+      "learning_rate": 3.023523985239853e-06,
+      "loss": 1.4212,
+      "step": 1113
+    },
+    {
+      "epoch": 1.3864343497199751,
+      "grad_norm": 2.7063710689544678,
+      "learning_rate": 3.021217712177122e-06,
+      "loss": 1.3769,
+      "step": 1114
+    },
+    {
+      "epoch": 1.387678904791537,
+      "grad_norm": 2.830411434173584,
+      "learning_rate": 3.0189114391143915e-06,
+      "loss": 1.4832,
+      "step": 1115
+    },
+    {
+      "epoch": 1.3889234598630988,
+      "grad_norm": 2.9075911045074463,
+      "learning_rate": 3.016605166051661e-06,
+      "loss": 1.486,
+      "step": 1116
+    },
+    {
+      "epoch": 1.390168014934661,
+      "grad_norm": 2.847822666168213,
+      "learning_rate": 3.01429889298893e-06,
+      "loss": 1.424,
+      "step": 1117
+    },
+    {
+      "epoch": 1.3914125700062228,
+      "grad_norm": 2.8822033405303955,
+      "learning_rate": 3.0119926199261993e-06,
+      "loss": 1.4218,
+      "step": 1118
+    },
+    {
+      "epoch": 1.3926571250777848,
+      "grad_norm": 2.8944740295410156,
+      "learning_rate": 3.009686346863469e-06,
+      "loss": 1.4283,
+      "step": 1119
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "grad_norm": 2.765190839767456,
+      "learning_rate": 3.007380073800738e-06,
+      "loss": 1.4622,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3939016801493467,
+      "eval_loss": 1.6451724767684937,
+      "eval_runtime": 43.7449,
+      "eval_samples_per_second": 22.86,
+      "eval_steps_per_second": 0.96,
+      "step": 1120
+    },
+    {
+      "epoch": 1.3951462352209085,
+      "grad_norm": 2.877002716064453,
+      "learning_rate": 3.0050738007380076e-06,
+      "loss": 1.4145,
+      "step": 1121
+    },
+    {
+      "epoch": 1.3963907902924704,
+      "grad_norm": 2.870389699935913,
+      "learning_rate": 3.002767527675277e-06,
+      "loss": 1.4401,
+      "step": 1122
+    },
+    {
+      "epoch": 1.3976353453640322,
+      "grad_norm": 2.7788920402526855,
+      "learning_rate": 3.000461254612546e-06,
+      "loss": 1.4514,
+      "step": 1123
+    },
+    {
+      "epoch": 1.3988799004355943,
+      "grad_norm": 2.9269332885742188,
+      "learning_rate": 2.998154981549816e-06,
+      "loss": 1.3961,
+      "step": 1124
+    },
+    {
+      "epoch": 1.4001244555071561,
+      "grad_norm": 3.013291835784912,
+      "learning_rate": 2.9958487084870853e-06,
+      "loss": 1.4597,
+      "step": 1125
+    },
+    {
+      "epoch": 1.4013690105787182,
+      "grad_norm": 2.9063880443573,
+      "learning_rate": 2.9935424354243547e-06,
+      "loss": 1.4668,
+      "step": 1126
+    },
+    {
+      "epoch": 1.40261356565028,
+      "grad_norm": 2.8926970958709717,
+      "learning_rate": 2.9912361623616237e-06,
+      "loss": 1.4332,
+      "step": 1127
+    },
+    {
+      "epoch": 1.403858120721842,
+      "grad_norm": 2.8563647270202637,
+      "learning_rate": 2.988929889298893e-06,
+      "loss": 1.407,
+      "step": 1128
+    },
+    {
+      "epoch": 1.4051026757934038,
+      "grad_norm": 2.8725359439849854,
+      "learning_rate": 2.986623616236163e-06,
+      "loss": 1.4434,
+      "step": 1129
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "grad_norm": 2.906898260116577,
+      "learning_rate": 2.984317343173432e-06,
+      "loss": 1.4745,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4063472308649658,
+      "eval_loss": 1.647884726524353,
+      "eval_runtime": 44.9221,
+      "eval_samples_per_second": 22.261,
+      "eval_steps_per_second": 0.935,
+      "step": 1130
+    },
+    {
+      "epoch": 1.4075917859365277,
+      "grad_norm": 2.8297719955444336,
+      "learning_rate": 2.9820110701107014e-06,
+      "loss": 1.4542,
+      "step": 1131
+    },
+    {
+      "epoch": 1.4088363410080897,
+      "grad_norm": 2.841031551361084,
+      "learning_rate": 2.979704797047971e-06,
+      "loss": 1.4105,
+      "step": 1132
+    },
+    {
+      "epoch": 1.4100808960796516,
+      "grad_norm": 2.867791175842285,
+      "learning_rate": 2.97739852398524e-06,
+      "loss": 1.4409,
+      "step": 1133
+    },
+    {
+      "epoch": 1.4113254511512134,
+      "grad_norm": 2.8156039714813232,
+      "learning_rate": 2.9750922509225093e-06,
+      "loss": 1.4284,
+      "step": 1134
+    },
+    {
+      "epoch": 1.4125700062227753,
+      "grad_norm": 2.825985908508301,
+      "learning_rate": 2.972785977859779e-06,
+      "loss": 1.4594,
+      "step": 1135
+    },
+    {
+      "epoch": 1.4138145612943374,
+      "grad_norm": 3.0134711265563965,
+      "learning_rate": 2.970479704797048e-06,
+      "loss": 1.4743,
+      "step": 1136
+    },
+    {
+      "epoch": 1.4150591163658992,
+      "grad_norm": 2.9829142093658447,
+      "learning_rate": 2.9681734317343175e-06,
+      "loss": 1.4935,
+      "step": 1137
+    },
+    {
+      "epoch": 1.416303671437461,
+      "grad_norm": 3.067620277404785,
+      "learning_rate": 2.965867158671587e-06,
+      "loss": 1.4266,
+      "step": 1138
+    },
+    {
+      "epoch": 1.4175482265090231,
+      "grad_norm": 2.8969919681549072,
+      "learning_rate": 2.9635608856088564e-06,
+      "loss": 1.451,
+      "step": 1139
+    },
+    {
+      "epoch": 1.418792781580585,
+      "grad_norm": 2.8706412315368652,
+      "learning_rate": 2.9612546125461254e-06,
+      "loss": 1.4834,
+      "step": 1140
+    },
+    {
+      "epoch": 1.418792781580585,
+      "eval_loss": 1.643480658531189,
+      "eval_runtime": 44.743,
+      "eval_samples_per_second": 22.35,
+      "eval_steps_per_second": 0.939,
+      "step": 1140
+    },
+    {
+      "epoch": 1.4200373366521468,
+      "grad_norm": 2.8801848888397217,
+      "learning_rate": 2.958948339483395e-06,
+      "loss": 1.4258,
+      "step": 1141
+    },
+    {
+      "epoch": 1.4212818917237087,
+      "grad_norm": 2.774635076522827,
+      "learning_rate": 2.9566420664206646e-06,
+      "loss": 1.416,
+      "step": 1142
+    },
+    {
+      "epoch": 1.4225264467952707,
+      "grad_norm": 2.8460140228271484,
+      "learning_rate": 2.9543357933579336e-06,
+      "loss": 1.4213,
+      "step": 1143
+    },
+    {
+      "epoch": 1.4237710018668326,
+      "grad_norm": 2.907888889312744,
+      "learning_rate": 2.952029520295203e-06,
+      "loss": 1.4507,
+      "step": 1144
+    },
+    {
+      "epoch": 1.4250155569383947,
+      "grad_norm": 2.789041757583618,
+      "learning_rate": 2.949723247232473e-06,
+      "loss": 1.4257,
+      "step": 1145
+    },
+    {
+      "epoch": 1.4262601120099565,
+      "grad_norm": 2.78859806060791,
+      "learning_rate": 2.947416974169742e-06,
+      "loss": 1.4377,
+      "step": 1146
+    },
+    {
+      "epoch": 1.4275046670815184,
+      "grad_norm": 2.840303421020508,
+      "learning_rate": 2.9451107011070113e-06,
+      "loss": 1.4684,
+      "step": 1147
+    },
+    {
+      "epoch": 1.4287492221530802,
+      "grad_norm": 2.8800530433654785,
+      "learning_rate": 2.9428044280442807e-06,
+      "loss": 1.4912,
+      "step": 1148
+    },
+    {
+      "epoch": 1.4299937772246423,
+      "grad_norm": 2.841092824935913,
+      "learning_rate": 2.9404981549815497e-06,
+      "loss": 1.4234,
+      "step": 1149
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "grad_norm": 2.8226537704467773,
+      "learning_rate": 2.938191881918819e-06,
+      "loss": 1.4578,
+      "step": 1150
+    },
+    {
+      "epoch": 1.4312383322962041,
+      "eval_loss": 1.643249273300171,
+      "eval_runtime": 43.5586,
+      "eval_samples_per_second": 22.958,
+      "eval_steps_per_second": 0.964,
+      "step": 1150
+    },
+    {
+      "epoch": 1.432482887367766,
+      "grad_norm": 2.869736909866333,
+      "learning_rate": 2.935885608856089e-06,
+      "loss": 1.4701,
+      "step": 1151
+    },
+    {
+      "epoch": 1.433727442439328,
+      "grad_norm": 2.689211130142212,
+      "learning_rate": 2.9335793357933584e-06,
+      "loss": 1.4483,
+      "step": 1152
+    },
+    {
+      "epoch": 1.4349719975108899,
+      "grad_norm": 2.9267077445983887,
+      "learning_rate": 2.9312730627306274e-06,
+      "loss": 1.4446,
+      "step": 1153
+    },
+    {
+      "epoch": 1.4362165525824517,
+      "grad_norm": 2.906862497329712,
+      "learning_rate": 2.928966789667897e-06,
+      "loss": 1.4524,
+      "step": 1154
+    },
+    {
+      "epoch": 1.4374611076540136,
+      "grad_norm": 2.8249800205230713,
+      "learning_rate": 2.9266605166051663e-06,
+      "loss": 1.4208,
+      "step": 1155
+    },
+    {
+      "epoch": 1.4387056627255757,
+      "grad_norm": 2.9836032390594482,
+      "learning_rate": 2.9243542435424353e-06,
+      "loss": 1.4581,
+      "step": 1156
+    },
+    {
+      "epoch": 1.4399502177971375,
+      "grad_norm": 2.9595284461975098,
+      "learning_rate": 2.922047970479705e-06,
+      "loss": 1.4141,
+      "step": 1157
+    },
+    {
+      "epoch": 1.4411947728686996,
+      "grad_norm": 3.0363423824310303,
+      "learning_rate": 2.9197416974169746e-06,
+      "loss": 1.4859,
+      "step": 1158
+    },
+    {
+      "epoch": 1.4424393279402614,
+      "grad_norm": 2.911201000213623,
+      "learning_rate": 2.9174354243542436e-06,
+      "loss": 1.4346,
+      "step": 1159
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "grad_norm": 2.8617193698883057,
+      "learning_rate": 2.915129151291513e-06,
+      "loss": 1.433,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4436838830118233,
+      "eval_loss": 1.6437232494354248,
+      "eval_runtime": 43.961,
+      "eval_samples_per_second": 22.747,
+      "eval_steps_per_second": 0.955,
+      "step": 1160
+    },
+    {
+      "epoch": 1.4449284380833851,
+      "grad_norm": 2.9584295749664307,
+      "learning_rate": 2.912822878228783e-06,
+      "loss": 1.4847,
+      "step": 1161
+    },
+    {
+      "epoch": 1.4461729931549472,
+      "grad_norm": 2.8209400177001953,
+      "learning_rate": 2.9105166051660522e-06,
+      "loss": 1.421,
+      "step": 1162
+    },
+    {
+      "epoch": 1.447417548226509,
+      "grad_norm": 2.847637414932251,
+      "learning_rate": 2.9082103321033212e-06,
+      "loss": 1.4326,
+      "step": 1163
+    },
+    {
+      "epoch": 1.4486621032980709,
+      "grad_norm": 2.836228132247925,
+      "learning_rate": 2.9059040590405907e-06,
+      "loss": 1.4459,
+      "step": 1164
+    },
+    {
+      "epoch": 1.449906658369633,
+      "grad_norm": 2.8327207565307617,
+      "learning_rate": 2.90359778597786e-06,
+      "loss": 1.4189,
+      "step": 1165
+    },
+    {
+      "epoch": 1.4511512134411948,
+      "grad_norm": 2.880643606185913,
+      "learning_rate": 2.901291512915129e-06,
+      "loss": 1.4214,
+      "step": 1166
+    },
+    {
+      "epoch": 1.4523957685127566,
+      "grad_norm": 2.9637348651885986,
+      "learning_rate": 2.898985239852399e-06,
+      "loss": 1.4471,
+      "step": 1167
+    },
+    {
+      "epoch": 1.4536403235843185,
+      "grad_norm": 3.006145477294922,
+      "learning_rate": 2.8966789667896684e-06,
+      "loss": 1.4228,
+      "step": 1168
+    },
+    {
+      "epoch": 1.4548848786558806,
+      "grad_norm": 2.8547627925872803,
+      "learning_rate": 2.8943726937269374e-06,
+      "loss": 1.4707,
+      "step": 1169
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "grad_norm": 2.973092794418335,
+      "learning_rate": 2.8920664206642068e-06,
+      "loss": 1.4804,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4561294337274424,
+      "eval_loss": 1.6340677738189697,
+      "eval_runtime": 44.6094,
+      "eval_samples_per_second": 22.417,
+      "eval_steps_per_second": 0.942,
+      "step": 1170
+    },
+    {
+      "epoch": 1.4573739887990045,
+      "grad_norm": 2.9433603286743164,
+      "learning_rate": 2.889760147601476e-06,
+      "loss": 1.4354,
+      "step": 1171
+    },
+    {
+      "epoch": 1.4586185438705663,
+      "grad_norm": 2.872236728668213,
+      "learning_rate": 2.887453874538745e-06,
+      "loss": 1.418,
+      "step": 1172
+    },
+    {
+      "epoch": 1.4598630989421282,
+      "grad_norm": 2.847557783126831,
+      "learning_rate": 2.885147601476015e-06,
+      "loss": 1.4306,
+      "step": 1173
+    },
+    {
+      "epoch": 1.46110765401369,
+      "grad_norm": 2.7554969787597656,
+      "learning_rate": 2.8828413284132845e-06,
+      "loss": 1.4295,
+      "step": 1174
+    },
+    {
+      "epoch": 1.462352209085252,
+      "grad_norm": 2.8754842281341553,
+      "learning_rate": 2.880535055350554e-06,
+      "loss": 1.4482,
+      "step": 1175
+    },
+    {
+      "epoch": 1.463596764156814,
+      "grad_norm": 3.0115292072296143,
+      "learning_rate": 2.878228782287823e-06,
+      "loss": 1.4428,
+      "step": 1176
+    },
+    {
+      "epoch": 1.4648413192283758,
+      "grad_norm": 2.8976168632507324,
+      "learning_rate": 2.8759225092250923e-06,
+      "loss": 1.4707,
+      "step": 1177
+    },
+    {
+      "epoch": 1.4660858742999379,
+      "grad_norm": 2.7981812953948975,
+      "learning_rate": 2.873616236162362e-06,
+      "loss": 1.4693,
+      "step": 1178
+    },
+    {
+      "epoch": 1.4673304293714997,
+      "grad_norm": 2.8562371730804443,
+      "learning_rate": 2.871309963099631e-06,
+      "loss": 1.4242,
+      "step": 1179
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "grad_norm": 2.8705570697784424,
+      "learning_rate": 2.8690036900369006e-06,
+      "loss": 1.4725,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4685749844430616,
+      "eval_loss": 1.6338286399841309,
+      "eval_runtime": 44.7171,
+      "eval_samples_per_second": 22.363,
+      "eval_steps_per_second": 0.939,
+      "step": 1180
+    },
+    {
+      "epoch": 1.4698195395146234,
+      "grad_norm": 2.8025710582733154,
+      "learning_rate": 2.86669741697417e-06,
+      "loss": 1.427,
+      "step": 1181
+    },
+    {
+      "epoch": 1.4710640945861855,
+      "grad_norm": 2.845151901245117,
+      "learning_rate": 2.864391143911439e-06,
+      "loss": 1.4402,
+      "step": 1182
+    },
+    {
+      "epoch": 1.4723086496577473,
+      "grad_norm": 2.943373680114746,
+      "learning_rate": 2.862084870848709e-06,
+      "loss": 1.5194,
+      "step": 1183
+    },
+    {
+      "epoch": 1.4735532047293094,
+      "grad_norm": 2.896955966949463,
+      "learning_rate": 2.8597785977859783e-06,
+      "loss": 1.4242,
+      "step": 1184
+    },
+    {
+      "epoch": 1.4747977598008712,
+      "grad_norm": 3.131305456161499,
+      "learning_rate": 2.8574723247232473e-06,
+      "loss": 1.5008,
+      "step": 1185
+    },
+    {
+      "epoch": 1.476042314872433,
+      "grad_norm": 2.8605735301971436,
+      "learning_rate": 2.8551660516605167e-06,
+      "loss": 1.3533,
+      "step": 1186
+    },
+    {
+      "epoch": 1.477286869943995,
+      "grad_norm": 2.9158811569213867,
+      "learning_rate": 2.852859778597786e-06,
+      "loss": 1.4404,
+      "step": 1187
+    },
+    {
+      "epoch": 1.478531425015557,
+      "grad_norm": 2.7795321941375732,
+      "learning_rate": 2.850553505535056e-06,
+      "loss": 1.3977,
+      "step": 1188
+    },
+    {
+      "epoch": 1.4797759800871189,
+      "grad_norm": 2.8194212913513184,
+      "learning_rate": 2.848247232472325e-06,
+      "loss": 1.4111,
+      "step": 1189
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "grad_norm": 2.983750104904175,
+      "learning_rate": 2.8459409594095944e-06,
+      "loss": 1.4061,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4810205351586807,
+      "eval_loss": 1.6375409364700317,
+      "eval_runtime": 45.4026,
+      "eval_samples_per_second": 22.025,
+      "eval_steps_per_second": 0.925,
+      "step": 1190
+    },
+    {
+      "epoch": 1.4822650902302428,
+      "grad_norm": 3.0246665477752686,
+      "learning_rate": 2.843634686346864e-06,
+      "loss": 1.4532,
+      "step": 1191
+    },
+    {
+      "epoch": 1.4835096453018046,
+      "grad_norm": 2.8856449127197266,
+      "learning_rate": 2.841328413284133e-06,
+      "loss": 1.4528,
+      "step": 1192
+    },
+    {
+      "epoch": 1.4847542003733665,
+      "grad_norm": 2.848987102508545,
+      "learning_rate": 2.8390221402214022e-06,
+      "loss": 1.4369,
+      "step": 1193
+    },
+    {
+      "epoch": 1.4859987554449283,
+      "grad_norm": 2.970419406890869,
+      "learning_rate": 2.836715867158672e-06,
+      "loss": 1.4605,
+      "step": 1194
+    },
+    {
+      "epoch": 1.4872433105164904,
+      "grad_norm": 3.156613826751709,
+      "learning_rate": 2.834409594095941e-06,
+      "loss": 1.4489,
+      "step": 1195
+    },
+    {
+      "epoch": 1.4884878655880522,
+      "grad_norm": 3.095349073410034,
+      "learning_rate": 2.8321033210332105e-06,
+      "loss": 1.4604,
+      "step": 1196
+    },
+    {
+      "epoch": 1.4897324206596143,
+      "grad_norm": 2.870742082595825,
+      "learning_rate": 2.82979704797048e-06,
+      "loss": 1.4284,
+      "step": 1197
+    },
+    {
+      "epoch": 1.4909769757311762,
+      "grad_norm": 2.9415056705474854,
+      "learning_rate": 2.827490774907749e-06,
+      "loss": 1.4715,
+      "step": 1198
+    },
+    {
+      "epoch": 1.492221530802738,
+      "grad_norm": 2.8297767639160156,
+      "learning_rate": 2.8251845018450188e-06,
+      "loss": 1.4508,
+      "step": 1199
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "grad_norm": 2.872086524963379,
+      "learning_rate": 2.822878228782288e-06,
+      "loss": 1.5098,
+      "step": 1200
+    },
+    {
+      "epoch": 1.4934660858742999,
+      "eval_loss": 1.6357526779174805,
+      "eval_runtime": 51.422,
+      "eval_samples_per_second": 19.447,
+      "eval_steps_per_second": 0.817,
+      "step": 1200
+    },
+    {
+      "epoch": 1.494710640945862,
+      "grad_norm": 2.7152082920074463,
+      "learning_rate": 2.8205719557195576e-06,
+      "loss": 1.4329,
+      "step": 1201
+    },
+    {
+      "epoch": 1.4959551960174238,
+      "grad_norm": 2.866170883178711,
+      "learning_rate": 2.8182656826568266e-06,
+      "loss": 1.4455,
+      "step": 1202
+    },
+    {
+      "epoch": 1.4971997510889856,
+      "grad_norm": 2.9010684490203857,
+      "learning_rate": 2.815959409594096e-06,
+      "loss": 1.423,
+      "step": 1203
+    },
+    {
+      "epoch": 1.4984443061605477,
+      "grad_norm": 2.8588502407073975,
+      "learning_rate": 2.813653136531366e-06,
+      "loss": 1.4041,
+      "step": 1204
+    },
+    {
+      "epoch": 1.4996888612321095,
+      "grad_norm": 2.932544231414795,
+      "learning_rate": 2.811346863468635e-06,
+      "loss": 1.4559,
+      "step": 1205
+    },
+    {
+      "epoch": 1.5009334163036714,
+      "grad_norm": 2.7486014366149902,
+      "learning_rate": 2.8090405904059043e-06,
+      "loss": 1.4337,
+      "step": 1206
+    },
+    {
+      "epoch": 1.5021779713752332,
+      "grad_norm": 3.0866589546203613,
+      "learning_rate": 2.8067343173431737e-06,
+      "loss": 1.4186,
+      "step": 1207
+    },
+    {
+      "epoch": 1.5034225264467953,
+      "grad_norm": 2.9710276126861572,
+      "learning_rate": 2.8044280442804427e-06,
+      "loss": 1.419,
+      "step": 1208
+    },
+    {
+      "epoch": 1.5046670815183572,
+      "grad_norm": 2.8379950523376465,
+      "learning_rate": 2.802121771217712e-06,
+      "loss": 1.4922,
+      "step": 1209
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "grad_norm": 2.939629554748535,
+      "learning_rate": 2.799815498154982e-06,
+      "loss": 1.4402,
+      "step": 1210
+    },
+    {
+      "epoch": 1.5059116365899192,
+      "eval_loss": 1.6419442892074585,
+      "eval_runtime": 43.1202,
+      "eval_samples_per_second": 23.191,
+      "eval_steps_per_second": 0.974,
+      "step": 1210
+    },
+    {
+      "epoch": 1.507156191661481,
+      "grad_norm": 3.041389226913452,
+      "learning_rate": 2.7975092250922514e-06,
+      "loss": 1.4873,
+      "step": 1211
+    },
+    {
+      "epoch": 1.508400746733043,
+      "grad_norm": 2.9776995182037354,
+      "learning_rate": 2.7952029520295204e-06,
+      "loss": 1.4422,
+      "step": 1212
+    },
+    {
+      "epoch": 1.5096453018046048,
+      "grad_norm": 2.8798792362213135,
+      "learning_rate": 2.79289667896679e-06,
+      "loss": 1.4535,
+      "step": 1213
+    },
+    {
+      "epoch": 1.5108898568761666,
+      "grad_norm": 2.9016385078430176,
+      "learning_rate": 2.7905904059040597e-06,
+      "loss": 1.4462,
+      "step": 1214
+    },
+    {
+      "epoch": 1.5121344119477287,
+      "grad_norm": 2.8987581729888916,
+      "learning_rate": 2.7882841328413287e-06,
+      "loss": 1.4477,
+      "step": 1215
+    },
+    {
+      "epoch": 1.5133789670192908,
+      "grad_norm": 2.960266351699829,
+      "learning_rate": 2.785977859778598e-06,
+      "loss": 1.4811,
+      "step": 1216
+    },
+    {
+      "epoch": 1.5146235220908526,
+      "grad_norm": 2.8965611457824707,
+      "learning_rate": 2.7836715867158675e-06,
+      "loss": 1.4366,
+      "step": 1217
+    },
+    {
+      "epoch": 1.5158680771624145,
+      "grad_norm": 3.0667364597320557,
+      "learning_rate": 2.7813653136531365e-06,
+      "loss": 1.3828,
+      "step": 1218
+    },
+    {
+      "epoch": 1.5171126322339763,
+      "grad_norm": 2.952362537384033,
+      "learning_rate": 2.779059040590406e-06,
+      "loss": 1.3903,
+      "step": 1219
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "grad_norm": 2.9899795055389404,
+      "learning_rate": 2.776752767527676e-06,
+      "loss": 1.4312,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5183571873055381,
+      "eval_loss": 1.640383005142212,
+      "eval_runtime": 45.4128,
+      "eval_samples_per_second": 22.02,
+      "eval_steps_per_second": 0.925,
+      "step": 1220
+    },
+    {
+      "epoch": 1.5196017423771002,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4344,
+      "step": 1221
+    },
+    {
+      "epoch": 1.520846297448662,
+      "grad_norm": 3.2202725410461426,
+      "learning_rate": 2.774446494464945e-06,
+      "loss": 1.4386,
+      "step": 1222
+    },
+    {
+      "epoch": 1.5220908525202241,
+      "grad_norm": 2.883223295211792,
+      "learning_rate": 2.7721402214022142e-06,
+      "loss": 1.4401,
+      "step": 1223
+    },
+    {
+      "epoch": 1.523335407591786,
+      "grad_norm": 2.8346424102783203,
+      "learning_rate": 2.7698339483394837e-06,
+      "loss": 1.3926,
+      "step": 1224
+    },
+    {
+      "epoch": 1.5245799626633478,
+      "grad_norm": 2.8898870944976807,
+      "learning_rate": 2.767527675276753e-06,
+      "loss": 1.4128,
+      "step": 1225
+    },
+    {
+      "epoch": 1.5258245177349097,
+      "grad_norm": 2.828594923019409,
+      "learning_rate": 2.765221402214022e-06,
+      "loss": 1.4103,
+      "step": 1226
+    },
+    {
+      "epoch": 1.5270690728064715,
+      "grad_norm": 2.882558822631836,
+      "learning_rate": 2.762915129151292e-06,
+      "loss": 1.4186,
+      "step": 1227
+    },
+    {
+      "epoch": 1.5283136278780336,
+      "grad_norm": 2.955760955810547,
+      "learning_rate": 2.7606088560885613e-06,
+      "loss": 1.3705,
+      "step": 1228
+    },
+    {
+      "epoch": 1.5295581829495957,
+      "grad_norm": 2.862934112548828,
+      "learning_rate": 2.7583025830258303e-06,
+      "loss": 1.5012,
+      "step": 1229
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "grad_norm": 2.926501989364624,
+      "learning_rate": 2.7559963099630998e-06,
+      "loss": 1.3875,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5308027380211575,
+      "eval_loss": 1.6376254558563232,
+      "eval_runtime": 47.9535,
+      "eval_samples_per_second": 20.854,
+      "eval_steps_per_second": 0.876,
+      "step": 1230
+    },
+    {
+      "epoch": 1.5320472930927194,
+      "grad_norm": 2.7648823261260986,
+      "learning_rate": 2.753690036900369e-06,
+      "loss": 1.3781,
+      "step": 1231
+    },
+    {
+      "epoch": 1.5332918481642812,
+      "grad_norm": 2.927103281021118,
+      "learning_rate": 2.751383763837638e-06,
+      "loss": 1.4091,
+      "step": 1232
+    },
+    {
+      "epoch": 1.534536403235843,
+      "grad_norm": 2.964823007583618,
+      "learning_rate": 2.749077490774908e-06,
+      "loss": 1.5075,
+      "step": 1233
+    },
+    {
+      "epoch": 1.5357809583074051,
+      "grad_norm": 2.7944016456604004,
+      "learning_rate": 2.7467712177121775e-06,
+      "loss": 1.3888,
+      "step": 1234
+    },
+    {
+      "epoch": 1.537025513378967,
+      "grad_norm": 2.891592025756836,
+      "learning_rate": 2.7444649446494465e-06,
+      "loss": 1.4825,
+      "step": 1235
+    },
+    {
+      "epoch": 1.538270068450529,
+      "grad_norm": 2.8120031356811523,
+      "learning_rate": 2.742158671586716e-06,
+      "loss": 1.4246,
+      "step": 1236
+    },
+    {
+      "epoch": 1.539514623522091,
+      "grad_norm": 2.9384255409240723,
+      "learning_rate": 2.7398523985239857e-06,
+      "loss": 1.4657,
+      "step": 1237
+    },
+    {
+      "epoch": 1.5407591785936527,
+      "grad_norm": 2.73111891746521,
+      "learning_rate": 2.737546125461255e-06,
+      "loss": 1.4689,
+      "step": 1238
+    },
+    {
+      "epoch": 1.5420037336652146,
+      "grad_norm": 2.858963966369629,
+      "learning_rate": 2.735239852398524e-06,
+      "loss": 1.4626,
+      "step": 1239
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "grad_norm": 2.9148507118225098,
+      "learning_rate": 2.7329335793357936e-06,
+      "loss": 1.4787,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5432482887367764,
+      "eval_loss": 1.6436412334442139,
+      "eval_runtime": 47.3139,
+      "eval_samples_per_second": 21.135,
+      "eval_steps_per_second": 0.888,
+      "step": 1240
+    },
+    {
+      "epoch": 1.5444928438083385,
+      "grad_norm": 2.8731327056884766,
+      "learning_rate": 2.730627306273063e-06,
+      "loss": 1.4748,
+      "step": 1241
+    },
+    {
+      "epoch": 1.5457373988799006,
+      "grad_norm": 2.990265369415283,
+      "learning_rate": 2.728321033210332e-06,
+      "loss": 1.4375,
+      "step": 1242
+    },
+    {
+      "epoch": 1.5469819539514624,
+      "grad_norm": 2.874321460723877,
+      "learning_rate": 2.726014760147602e-06,
+      "loss": 1.4154,
+      "step": 1243
+    },
+    {
+      "epoch": 1.5482265090230243,
+      "grad_norm": 2.9316225051879883,
+      "learning_rate": 2.7237084870848713e-06,
+      "loss": 1.4285,
+      "step": 1244
+    },
+    {
+      "epoch": 1.5494710640945861,
+      "grad_norm": 2.783966302871704,
+      "learning_rate": 2.7214022140221403e-06,
+      "loss": 1.409,
+      "step": 1245
+    },
+    {
+      "epoch": 1.550715619166148,
+      "grad_norm": 2.9578707218170166,
+      "learning_rate": 2.7190959409594097e-06,
+      "loss": 1.4325,
+      "step": 1246
+    },
+    {
+      "epoch": 1.55196017423771,
+      "grad_norm": 2.869893789291382,
+      "learning_rate": 2.716789667896679e-06,
+      "loss": 1.3989,
+      "step": 1247
+    },
+    {
+      "epoch": 1.553204729309272,
+      "grad_norm": 2.962944269180298,
+      "learning_rate": 2.714483394833948e-06,
+      "loss": 1.4302,
+      "step": 1248
+    },
+    {
+      "epoch": 1.554449284380834,
+      "grad_norm": 2.974494695663452,
+      "learning_rate": 2.712177121771218e-06,
+      "loss": 1.46,
+      "step": 1249
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "grad_norm": 2.9154603481292725,
+      "learning_rate": 2.7098708487084874e-06,
+      "loss": 1.4388,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5556938394523958,
+      "eval_loss": 1.6357773542404175,
+      "eval_runtime": 47.6085,
+      "eval_samples_per_second": 21.005,
+      "eval_steps_per_second": 0.882,
+      "step": 1250
+    },
+    {
+      "epoch": 1.5569383945239577,
+      "grad_norm": 3.0333547592163086,
+      "learning_rate": 2.707564575645757e-06,
+      "loss": 1.4846,
+      "step": 1251
+    },
+    {
+      "epoch": 1.5581829495955195,
+      "grad_norm": 3.06097674369812,
+      "learning_rate": 2.705258302583026e-06,
+      "loss": 1.4355,
+      "step": 1252
+    },
+    {
+      "epoch": 1.5594275046670814,
+      "grad_norm": 2.872680425643921,
+      "learning_rate": 2.7029520295202956e-06,
+      "loss": 1.4219,
+      "step": 1253
+    },
+    {
+      "epoch": 1.5606720597386434,
+      "grad_norm": 2.9066731929779053,
+      "learning_rate": 2.700645756457565e-06,
+      "loss": 1.384,
+      "step": 1254
+    },
+    {
+      "epoch": 1.5619166148102055,
+      "grad_norm": 3.0570194721221924,
+      "learning_rate": 2.698339483394834e-06,
+      "loss": 1.4687,
+      "step": 1255
+    },
+    {
+      "epoch": 1.5631611698817673,
+      "grad_norm": 3.1955857276916504,
+      "learning_rate": 2.6960332103321035e-06,
+      "loss": 1.4736,
+      "step": 1256
+    },
+    {
+      "epoch": 1.5644057249533292,
+      "grad_norm": 2.9686198234558105,
+      "learning_rate": 2.693726937269373e-06,
+      "loss": 1.4176,
+      "step": 1257
+    },
+    {
+      "epoch": 1.565650280024891,
+      "grad_norm": 3.1161272525787354,
+      "learning_rate": 2.691420664206642e-06,
+      "loss": 1.443,
+      "step": 1258
+    },
+    {
+      "epoch": 1.5668948350964529,
+      "grad_norm": 2.9031193256378174,
+      "learning_rate": 2.6891143911439118e-06,
+      "loss": 1.3871,
+      "step": 1259
+    },
+    {
+      "epoch": 1.568139390168015,
+      "grad_norm": 2.9120476245880127,
+      "learning_rate": 2.686808118081181e-06,
+      "loss": 1.4406,
+      "step": 1260
+    },
+    {
+      "epoch": 1.568139390168015,
+      "eval_loss": 1.6291446685791016,
+      "eval_runtime": 49.7006,
+      "eval_samples_per_second": 20.12,
+      "eval_steps_per_second": 0.845,
+      "step": 1260
+    },
+    {
+      "epoch": 1.5693839452395768,
+      "grad_norm": 3.3181824684143066,
+      "learning_rate": 2.6845018450184506e-06,
+      "loss": 1.5278,
+      "step": 1261
+    },
+    {
+      "epoch": 1.5706285003111389,
+      "grad_norm": 2.9628148078918457,
+      "learning_rate": 2.6821955719557196e-06,
+      "loss": 1.4206,
+      "step": 1262
+    },
+    {
+      "epoch": 1.5718730553827007,
+      "grad_norm": 2.9828474521636963,
+      "learning_rate": 2.679889298892989e-06,
+      "loss": 1.5478,
+      "step": 1263
+    },
+    {
+      "epoch": 1.5731176104542626,
+      "grad_norm": 3.018711805343628,
+      "learning_rate": 2.677583025830259e-06,
+      "loss": 1.4442,
+      "step": 1264
+    },
+    {
+      "epoch": 1.5743621655258244,
+      "grad_norm": 2.8596792221069336,
+      "learning_rate": 2.675276752767528e-06,
+      "loss": 1.4325,
+      "step": 1265
+    },
+    {
+      "epoch": 1.5756067205973865,
+      "grad_norm": 2.9592769145965576,
+      "learning_rate": 2.6729704797047973e-06,
+      "loss": 1.4059,
+      "step": 1266
+    },
+    {
+      "epoch": 1.5768512756689483,
+      "grad_norm": 2.773589611053467,
+      "learning_rate": 2.6706642066420667e-06,
+      "loss": 1.3608,
+      "step": 1267
+    },
+    {
+      "epoch": 1.5780958307405104,
+      "grad_norm": 3.313255548477173,
+      "learning_rate": 2.6683579335793357e-06,
+      "loss": 1.4552,
+      "step": 1268
+    },
+    {
+      "epoch": 1.5793403858120723,
+      "grad_norm": 2.977649211883545,
+      "learning_rate": 2.6660516605166056e-06,
+      "loss": 1.4221,
+      "step": 1269
+    },
+    {
+      "epoch": 1.580584940883634,
+      "grad_norm": 3.0672051906585693,
+      "learning_rate": 2.663745387453875e-06,
+      "loss": 1.4568,
+      "step": 1270
+    },
+    {
+      "epoch": 1.580584940883634,
+      "eval_loss": 1.6317907571792603,
+      "eval_runtime": 47.2354,
+      "eval_samples_per_second": 21.171,
+      "eval_steps_per_second": 0.889,
+      "step": 1270
+    },
+    {
+      "epoch": 1.581829495955196,
+      "grad_norm": 3.029320240020752,
+      "learning_rate": 2.661439114391144e-06,
+      "loss": 1.4432,
+      "step": 1271
+    },
+    {
+      "epoch": 1.5830740510267578,
+      "grad_norm": 3.0183846950531006,
+      "learning_rate": 2.6591328413284134e-06,
+      "loss": 1.4332,
+      "step": 1272
+    },
+    {
+      "epoch": 1.5843186060983199,
+      "grad_norm": 2.856551170349121,
+      "learning_rate": 2.656826568265683e-06,
+      "loss": 1.4291,
+      "step": 1273
+    },
+    {
+      "epoch": 1.5855631611698817,
+      "grad_norm": 3.124971389770508,
+      "learning_rate": 2.6545202952029527e-06,
+      "loss": 1.4363,
+      "step": 1274
+    },
+    {
+      "epoch": 1.5868077162414438,
+      "grad_norm": 2.9470791816711426,
+      "learning_rate": 2.6522140221402217e-06,
+      "loss": 1.4502,
+      "step": 1275
+    },
+    {
+      "epoch": 1.5880522713130056,
+      "grad_norm": 2.9116358757019043,
+      "learning_rate": 2.649907749077491e-06,
+      "loss": 1.4314,
+      "step": 1276
+    },
+    {
+      "epoch": 1.5892968263845675,
+      "grad_norm": 3.2575323581695557,
+      "learning_rate": 2.6476014760147605e-06,
+      "loss": 1.4749,
+      "step": 1277
+    },
+    {
+      "epoch": 1.5905413814561293,
+      "grad_norm": 2.935518503189087,
+      "learning_rate": 2.6452952029520295e-06,
+      "loss": 1.4265,
+      "step": 1278
+    },
+    {
+      "epoch": 1.5917859365276914,
+      "grad_norm": 2.9750072956085205,
+      "learning_rate": 2.642988929889299e-06,
+      "loss": 1.4501,
+      "step": 1279
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "grad_norm": 2.8086845874786377,
+      "learning_rate": 2.640682656826569e-06,
+      "loss": 1.4557,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5930304915992533,
+      "eval_loss": 1.6302106380462646,
+      "eval_runtime": 52.3962,
+      "eval_samples_per_second": 19.085,
+      "eval_steps_per_second": 0.802,
+      "step": 1280
+    },
+    {
+      "epoch": 1.5942750466708153,
+      "grad_norm": 2.741849422454834,
+      "learning_rate": 2.638376383763838e-06,
+      "loss": 1.3918,
+      "step": 1281
+    },
+    {
+      "epoch": 1.5955196017423772,
+      "grad_norm": 2.9414806365966797,
+      "learning_rate": 2.6360701107011072e-06,
+      "loss": 1.3882,
+      "step": 1282
+    },
+    {
+      "epoch": 1.596764156813939,
+      "grad_norm": 2.901514768600464,
+      "learning_rate": 2.6337638376383766e-06,
+      "loss": 1.4331,
+      "step": 1283
+    },
+    {
+      "epoch": 1.5980087118855009,
+      "grad_norm": 2.8159122467041016,
+      "learning_rate": 2.6314575645756456e-06,
+      "loss": 1.4064,
+      "step": 1284
+    },
+    {
+      "epoch": 1.5992532669570627,
+      "grad_norm": 2.900561571121216,
+      "learning_rate": 2.629151291512915e-06,
+      "loss": 1.3731,
+      "step": 1285
+    },
+    {
+      "epoch": 1.6004978220286248,
+      "grad_norm": 3.019540548324585,
+      "learning_rate": 2.626845018450185e-06,
+      "loss": 1.4506,
+      "step": 1286
+    },
+    {
+      "epoch": 1.6017423771001866,
+      "grad_norm": 2.9263620376586914,
+      "learning_rate": 2.6245387453874543e-06,
+      "loss": 1.4754,
+      "step": 1287
+    },
+    {
+      "epoch": 1.6029869321717487,
+      "grad_norm": 2.8681881427764893,
+      "learning_rate": 2.6222324723247233e-06,
+      "loss": 1.4124,
+      "step": 1288
+    },
+    {
+      "epoch": 1.6042314872433105,
+      "grad_norm": 2.922945261001587,
+      "learning_rate": 2.6199261992619928e-06,
+      "loss": 1.5101,
+      "step": 1289
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "grad_norm": 2.9548606872558594,
+      "learning_rate": 2.6176199261992626e-06,
+      "loss": 1.41,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6054760423148724,
+      "eval_loss": 1.6357502937316895,
+      "eval_runtime": 46.9547,
+      "eval_samples_per_second": 21.297,
+      "eval_steps_per_second": 0.894,
+      "step": 1290
+    },
+    {
+      "epoch": 1.6067205973864342,
+      "grad_norm": 2.837184190750122,
+      "learning_rate": 2.6153136531365316e-06,
+      "loss": 1.4354,
+      "step": 1291
+    },
+    {
+      "epoch": 1.6079651524579963,
+      "grad_norm": 2.818990468978882,
+      "learning_rate": 2.613007380073801e-06,
+      "loss": 1.4474,
+      "step": 1292
+    },
+    {
+      "epoch": 1.6092097075295582,
+      "grad_norm": 2.7880146503448486,
+      "learning_rate": 2.6107011070110704e-06,
+      "loss": 1.4156,
+      "step": 1293
+    },
+    {
+      "epoch": 1.6104542626011202,
+      "grad_norm": 2.790971040725708,
+      "learning_rate": 2.6083948339483394e-06,
+      "loss": 1.4671,
+      "step": 1294
+    },
+    {
+      "epoch": 1.611698817672682,
+      "grad_norm": 2.943129777908325,
+      "learning_rate": 2.606088560885609e-06,
+      "loss": 1.4801,
+      "step": 1295
+    },
+    {
+      "epoch": 1.612943372744244,
+      "grad_norm": 2.8589110374450684,
+      "learning_rate": 2.6037822878228787e-06,
+      "loss": 1.4894,
+      "step": 1296
+    },
+    {
+      "epoch": 1.6141879278158058,
+      "grad_norm": 2.8929262161254883,
+      "learning_rate": 2.6014760147601477e-06,
+      "loss": 1.4022,
+      "step": 1297
+    },
+    {
+      "epoch": 1.6154324828873676,
+      "grad_norm": 2.970099925994873,
+      "learning_rate": 2.599169741697417e-06,
+      "loss": 1.5136,
+      "step": 1298
+    },
+    {
+      "epoch": 1.6166770379589297,
+      "grad_norm": 2.928466796875,
+      "learning_rate": 2.5968634686346866e-06,
+      "loss": 1.4743,
+      "step": 1299
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "grad_norm": 2.941580057144165,
+      "learning_rate": 2.594557195571956e-06,
+      "loss": 1.4397,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6179215930304915,
+      "eval_loss": 1.6348339319229126,
+      "eval_runtime": 50.433,
+      "eval_samples_per_second": 19.828,
+      "eval_steps_per_second": 0.833,
+      "step": 1300
+    },
+    {
+      "epoch": 1.6191661481020536,
+      "grad_norm": 2.903212308883667,
+      "learning_rate": 2.592250922509225e-06,
+      "loss": 1.4221,
+      "step": 1301
+    },
+    {
+      "epoch": 1.6204107031736155,
+      "grad_norm": 2.948651075363159,
+      "learning_rate": 2.589944649446495e-06,
+      "loss": 1.4363,
+      "step": 1302
+    },
+    {
+      "epoch": 1.6216552582451773,
+      "grad_norm": 2.8940505981445312,
+      "learning_rate": 2.5876383763837643e-06,
+      "loss": 1.398,
+      "step": 1303
+    },
+    {
+      "epoch": 1.6228998133167392,
+      "grad_norm": 2.950056552886963,
+      "learning_rate": 2.5853321033210333e-06,
+      "loss": 1.4425,
+      "step": 1304
+    },
+    {
+      "epoch": 1.6241443683883012,
+      "grad_norm": 2.9674088954925537,
+      "learning_rate": 2.5830258302583027e-06,
+      "loss": 1.4473,
+      "step": 1305
+    },
+    {
+      "epoch": 1.625388923459863,
+      "grad_norm": 2.7541685104370117,
+      "learning_rate": 2.5807195571955725e-06,
+      "loss": 1.4483,
+      "step": 1306
+    },
+    {
+      "epoch": 1.6266334785314251,
+      "grad_norm": 2.8897035121917725,
+      "learning_rate": 2.5784132841328415e-06,
+      "loss": 1.4572,
+      "step": 1307
+    },
+    {
+      "epoch": 1.627878033602987,
+      "grad_norm": 3.021559238433838,
+      "learning_rate": 2.576107011070111e-06,
+      "loss": 1.5164,
+      "step": 1308
+    },
+    {
+      "epoch": 1.6291225886745488,
+      "grad_norm": 2.884009838104248,
+      "learning_rate": 2.5738007380073804e-06,
+      "loss": 1.4196,
+      "step": 1309
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "grad_norm": 2.749497890472412,
+      "learning_rate": 2.5714944649446494e-06,
+      "loss": 1.4324,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6303671437461107,
+      "eval_loss": 1.6314265727996826,
+      "eval_runtime": 42.4647,
+      "eval_samples_per_second": 23.549,
+      "eval_steps_per_second": 0.989,
+      "step": 1310
+    },
+    {
+      "epoch": 1.6316116988176725,
+      "grad_norm": 2.8908045291900635,
+      "learning_rate": 2.5691881918819188e-06,
+      "loss": 1.4388,
+      "step": 1311
+    },
+    {
+      "epoch": 1.6328562538892346,
+      "grad_norm": 2.832808256149292,
+      "learning_rate": 2.5668819188191886e-06,
+      "loss": 1.4368,
+      "step": 1312
+    },
+    {
+      "epoch": 1.6341008089607967,
+      "grad_norm": 2.9047906398773193,
+      "learning_rate": 2.564575645756458e-06,
+      "loss": 1.4186,
+      "step": 1313
+    },
+    {
+      "epoch": 1.6353453640323585,
+      "grad_norm": 2.9271419048309326,
+      "learning_rate": 2.562269372693727e-06,
+      "loss": 1.3776,
+      "step": 1314
+    },
+    {
+      "epoch": 1.6365899191039204,
+      "grad_norm": 2.760314702987671,
+      "learning_rate": 2.5599630996309965e-06,
+      "loss": 1.386,
+      "step": 1315
+    },
+    {
+      "epoch": 1.6378344741754822,
+      "grad_norm": 2.9694864749908447,
+      "learning_rate": 2.557656826568266e-06,
+      "loss": 1.4542,
+      "step": 1316
+    },
+    {
+      "epoch": 1.639079029247044,
+      "grad_norm": 2.9950435161590576,
+      "learning_rate": 2.555350553505535e-06,
+      "loss": 1.4242,
+      "step": 1317
+    },
+    {
+      "epoch": 1.6403235843186061,
+      "grad_norm": 2.9291114807128906,
+      "learning_rate": 2.5530442804428047e-06,
+      "loss": 1.4289,
+      "step": 1318
+    },
+    {
+      "epoch": 1.641568139390168,
+      "grad_norm": 2.981961488723755,
+      "learning_rate": 2.550738007380074e-06,
+      "loss": 1.4283,
+      "step": 1319
+    },
+    {
+      "epoch": 1.64281269446173,
+      "grad_norm": 2.734123706817627,
+      "learning_rate": 2.548431734317343e-06,
+      "loss": 1.4144,
+      "step": 1320
+    },
+    {
+      "epoch": 1.64281269446173,
+      "eval_loss": 1.6249239444732666,
+      "eval_runtime": 48.8116,
+      "eval_samples_per_second": 20.487,
+      "eval_steps_per_second": 0.86,
+      "step": 1320
+    },
+    {
+      "epoch": 1.644057249533292,
+      "grad_norm": 2.9752919673919678,
+      "learning_rate": 2.5461254612546126e-06,
+      "loss": 1.3641,
+      "step": 1321
+    },
+    {
+      "epoch": 1.6453018046048538,
+      "grad_norm": 2.972463607788086,
+      "learning_rate": 2.543819188191882e-06,
+      "loss": 1.3797,
+      "step": 1322
+    },
+    {
+      "epoch": 1.6465463596764156,
+      "grad_norm": 2.905499219894409,
+      "learning_rate": 2.541512915129152e-06,
+      "loss": 1.4221,
+      "step": 1323
+    },
+    {
+      "epoch": 1.6477909147479775,
+      "grad_norm": 2.7921793460845947,
+      "learning_rate": 2.539206642066421e-06,
+      "loss": 1.4118,
+      "step": 1324
+    },
+    {
+      "epoch": 1.6490354698195395,
+      "grad_norm": 3.064558267593384,
+      "learning_rate": 2.5369003690036903e-06,
+      "loss": 1.4611,
+      "step": 1325
+    },
+    {
+      "epoch": 1.6502800248911016,
+      "grad_norm": 2.9690096378326416,
+      "learning_rate": 2.5345940959409597e-06,
+      "loss": 1.461,
+      "step": 1326
+    },
+    {
+      "epoch": 1.6515245799626634,
+      "grad_norm": 3.0379040241241455,
+      "learning_rate": 2.5322878228782287e-06,
+      "loss": 1.451,
+      "step": 1327
+    },
+    {
+      "epoch": 1.6527691350342253,
+      "grad_norm": 2.9426493644714355,
+      "learning_rate": 2.5299815498154986e-06,
+      "loss": 1.4671,
+      "step": 1328
+    },
+    {
+      "epoch": 1.6540136901057871,
+      "grad_norm": 2.915076732635498,
+      "learning_rate": 2.527675276752768e-06,
+      "loss": 1.4159,
+      "step": 1329
+    },
+    {
+      "epoch": 1.655258245177349,
+      "grad_norm": 2.8588879108428955,
+      "learning_rate": 2.525369003690037e-06,
+      "loss": 1.4978,
+      "step": 1330
+    },
+    {
+      "epoch": 1.655258245177349,
+      "eval_loss": 1.624009132385254,
+      "eval_runtime": 47.0461,
+      "eval_samples_per_second": 21.256,
+      "eval_steps_per_second": 0.893,
+      "step": 1330
+    },
+    {
+      "epoch": 1.656502800248911,
+      "grad_norm": 2.9942209720611572,
+      "learning_rate": 2.5230627306273064e-06,
+      "loss": 1.3872,
+      "step": 1331
+    },
+    {
+      "epoch": 1.657747355320473,
+      "grad_norm": 2.8424410820007324,
+      "learning_rate": 2.520756457564576e-06,
+      "loss": 1.4181,
+      "step": 1332
+    },
+    {
+      "epoch": 1.658991910392035,
+      "grad_norm": 2.9234843254089355,
+      "learning_rate": 2.518450184501845e-06,
+      "loss": 1.4516,
+      "step": 1333
+    },
+    {
+      "epoch": 1.6602364654635968,
+      "grad_norm": 2.8367815017700195,
+      "learning_rate": 2.5161439114391147e-06,
+      "loss": 1.4214,
+      "step": 1334
+    },
+    {
+      "epoch": 1.6614810205351587,
+      "grad_norm": 2.819149971008301,
+      "learning_rate": 2.513837638376384e-06,
+      "loss": 1.3883,
+      "step": 1335
+    },
+    {
+      "epoch": 1.6627255756067205,
+      "grad_norm": 3.024693250656128,
+      "learning_rate": 2.5115313653136535e-06,
+      "loss": 1.3902,
+      "step": 1336
+    },
+    {
+      "epoch": 1.6639701306782824,
+      "grad_norm": 2.903599262237549,
+      "learning_rate": 2.5092250922509225e-06,
+      "loss": 1.4238,
+      "step": 1337
+    },
+    {
+      "epoch": 1.6652146857498444,
+      "grad_norm": 2.8296589851379395,
+      "learning_rate": 2.506918819188192e-06,
+      "loss": 1.3993,
+      "step": 1338
+    },
+    {
+      "epoch": 1.6664592408214065,
+      "grad_norm": 2.873272657394409,
+      "learning_rate": 2.5046125461254618e-06,
+      "loss": 1.4186,
+      "step": 1339
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "grad_norm": 2.9268789291381836,
+      "learning_rate": 2.5023062730627308e-06,
+      "loss": 1.4242,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6677037958929684,
+      "eval_loss": 1.6260672807693481,
+      "eval_runtime": 47.5895,
+      "eval_samples_per_second": 21.013,
+      "eval_steps_per_second": 0.883,
+      "step": 1340
+    },
+    {
+      "epoch": 1.6689483509645302,
+      "grad_norm": 3.057659387588501,
+      "learning_rate": 2.5e-06,
+      "loss": 1.4077,
+      "step": 1341
+    },
+    {
+      "epoch": 1.670192906036092,
+      "grad_norm": 2.875095844268799,
+      "learning_rate": 2.4976937269372696e-06,
+      "loss": 1.394,
+      "step": 1342
+    },
+    {
+      "epoch": 1.671437461107654,
+      "grad_norm": 2.7579872608184814,
+      "learning_rate": 2.495387453874539e-06,
+      "loss": 1.4029,
+      "step": 1343
+    },
+    {
+      "epoch": 1.672682016179216,
+      "grad_norm": 2.8981001377105713,
+      "learning_rate": 2.4930811808118085e-06,
+      "loss": 1.4494,
+      "step": 1344
+    },
+    {
+      "epoch": 1.6739265712507778,
+      "grad_norm": 2.9696900844573975,
+      "learning_rate": 2.490774907749078e-06,
+      "loss": 1.4692,
+      "step": 1345
+    },
+    {
+      "epoch": 1.6751711263223399,
+      "grad_norm": 2.8094892501831055,
+      "learning_rate": 2.488468634686347e-06,
+      "loss": 1.441,
+      "step": 1346
+    },
+    {
+      "epoch": 1.6764156813939017,
+      "grad_norm": 2.8619563579559326,
+      "learning_rate": 2.4861623616236163e-06,
+      "loss": 1.4885,
+      "step": 1347
+    },
+    {
+      "epoch": 1.6776602364654636,
+      "grad_norm": 2.854151487350464,
+      "learning_rate": 2.4838560885608857e-06,
+      "loss": 1.4338,
+      "step": 1348
+    },
+    {
+      "epoch": 1.6789047915370254,
+      "grad_norm": 2.9707868099212646,
+      "learning_rate": 2.481549815498155e-06,
+      "loss": 1.4529,
+      "step": 1349
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "grad_norm": 3.0289969444274902,
+      "learning_rate": 2.4792435424354246e-06,
+      "loss": 1.4855,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6801493466085873,
+      "eval_loss": 1.6256393194198608,
+      "eval_runtime": 45.9524,
+      "eval_samples_per_second": 21.762,
+      "eval_steps_per_second": 0.914,
+      "step": 1350
+    },
+    {
+      "epoch": 1.6813939016801493,
+      "grad_norm": 2.8407845497131348,
+      "learning_rate": 2.476937269372694e-06,
+      "loss": 1.3876,
+      "step": 1351
+    },
+    {
+      "epoch": 1.6826384567517114,
+      "grad_norm": 2.9876599311828613,
+      "learning_rate": 2.4746309963099634e-06,
+      "loss": 1.3931,
+      "step": 1352
+    },
+    {
+      "epoch": 1.6838830118232733,
+      "grad_norm": 3.008683919906616,
+      "learning_rate": 2.472324723247233e-06,
+      "loss": 1.4482,
+      "step": 1353
+    },
+    {
+      "epoch": 1.6851275668948351,
+      "grad_norm": 2.8110079765319824,
+      "learning_rate": 2.470018450184502e-06,
+      "loss": 1.3851,
+      "step": 1354
+    },
+    {
+      "epoch": 1.686372121966397,
+      "grad_norm": 2.9402263164520264,
+      "learning_rate": 2.4677121771217713e-06,
+      "loss": 1.4009,
+      "step": 1355
+    },
+    {
+      "epoch": 1.6876166770379588,
+      "grad_norm": 2.9846489429473877,
+      "learning_rate": 2.4654059040590407e-06,
+      "loss": 1.3975,
+      "step": 1356
+    },
+    {
+      "epoch": 1.6888612321095209,
+      "grad_norm": 2.835801362991333,
+      "learning_rate": 2.46309963099631e-06,
+      "loss": 1.3421,
+      "step": 1357
+    },
+    {
+      "epoch": 1.6901057871810827,
+      "grad_norm": 2.9438889026641846,
+      "learning_rate": 2.4607933579335795e-06,
+      "loss": 1.4422,
+      "step": 1358
+    },
+    {
+      "epoch": 1.6913503422526448,
+      "grad_norm": 3.168829917907715,
+      "learning_rate": 2.458487084870849e-06,
+      "loss": 1.4596,
+      "step": 1359
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "grad_norm": 2.8412039279937744,
+      "learning_rate": 2.4561808118081184e-06,
+      "loss": 1.4263,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6925948973242066,
+      "eval_loss": 1.6265885829925537,
+      "eval_runtime": 50.3402,
+      "eval_samples_per_second": 19.865,
+      "eval_steps_per_second": 0.834,
+      "step": 1360
+    },
+    {
+      "epoch": 1.6938394523957685,
+      "grad_norm": 3.0540060997009277,
+      "learning_rate": 2.453874538745388e-06,
+      "loss": 1.4123,
+      "step": 1361
+    },
+    {
+      "epoch": 1.6950840074673303,
+      "grad_norm": 3.0544989109039307,
+      "learning_rate": 2.451568265682657e-06,
+      "loss": 1.412,
+      "step": 1362
+    },
+    {
+      "epoch": 1.6963285625388922,
+      "grad_norm": 3.114126205444336,
+      "learning_rate": 2.4492619926199267e-06,
+      "loss": 1.4952,
+      "step": 1363
+    },
+    {
+      "epoch": 1.6975731176104543,
+      "grad_norm": 2.9366016387939453,
+      "learning_rate": 2.4469557195571957e-06,
+      "loss": 1.5155,
+      "step": 1364
+    },
+    {
+      "epoch": 1.6988176726820163,
+      "grad_norm": 2.910677671432495,
+      "learning_rate": 2.444649446494465e-06,
+      "loss": 1.4393,
+      "step": 1365
+    },
+    {
+      "epoch": 1.7000622277535782,
+      "grad_norm": 2.8373727798461914,
+      "learning_rate": 2.4423431734317345e-06,
+      "loss": 1.4223,
+      "step": 1366
+    },
+    {
+      "epoch": 1.70130678282514,
+      "grad_norm": 2.865161895751953,
+      "learning_rate": 2.440036900369004e-06,
+      "loss": 1.4605,
+      "step": 1367
+    },
+    {
+      "epoch": 1.7025513378967019,
+      "grad_norm": 2.7929177284240723,
+      "learning_rate": 2.437730627306273e-06,
+      "loss": 1.4695,
+      "step": 1368
+    },
+    {
+      "epoch": 1.7037958929682637,
+      "grad_norm": 2.867953300476074,
+      "learning_rate": 2.4354243542435428e-06,
+      "loss": 1.3524,
+      "step": 1369
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "grad_norm": 2.8980207443237305,
+      "learning_rate": 2.4331180811808118e-06,
+      "loss": 1.4568,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7050404480398258,
+      "eval_loss": 1.6297556161880493,
+      "eval_runtime": 51.447,
+      "eval_samples_per_second": 19.437,
+      "eval_steps_per_second": 0.816,
+      "step": 1370
+    },
+    {
+      "epoch": 1.7062850031113876,
+      "grad_norm": 2.954972505569458,
+      "learning_rate": 2.4308118081180816e-06,
+      "loss": 1.4,
+      "step": 1371
+    },
+    {
+      "epoch": 1.7075295581829497,
+      "grad_norm": 2.973191738128662,
+      "learning_rate": 2.4285055350553506e-06,
+      "loss": 1.4062,
+      "step": 1372
+    },
+    {
+      "epoch": 1.7087741132545116,
+      "grad_norm": 2.951444387435913,
+      "learning_rate": 2.42619926199262e-06,
+      "loss": 1.4523,
+      "step": 1373
+    },
+    {
+      "epoch": 1.7100186683260734,
+      "grad_norm": 2.850445508956909,
+      "learning_rate": 2.4238929889298895e-06,
+      "loss": 1.4282,
+      "step": 1374
+    },
+    {
+      "epoch": 1.7112632233976353,
+      "grad_norm": 2.7643561363220215,
+      "learning_rate": 2.421586715867159e-06,
+      "loss": 1.3807,
+      "step": 1375
+    },
+    {
+      "epoch": 1.712507778469197,
+      "grad_norm": 2.8872134685516357,
+      "learning_rate": 2.4192804428044283e-06,
+      "loss": 1.4801,
+      "step": 1376
+    },
+    {
+      "epoch": 1.7137523335407592,
+      "grad_norm": 2.8748483657836914,
+      "learning_rate": 2.4169741697416977e-06,
+      "loss": 1.381,
+      "step": 1377
+    },
+    {
+      "epoch": 1.7149968886123212,
+      "grad_norm": 2.7833831310272217,
+      "learning_rate": 2.4146678966789667e-06,
+      "loss": 1.389,
+      "step": 1378
+    },
+    {
+      "epoch": 1.716241443683883,
+      "grad_norm": 2.994715690612793,
+      "learning_rate": 2.4123616236162366e-06,
+      "loss": 1.3917,
+      "step": 1379
+    },
+    {
+      "epoch": 1.717485998755445,
+      "grad_norm": 2.887026786804199,
+      "learning_rate": 2.4100553505535056e-06,
+      "loss": 1.3725,
+      "step": 1380
+    },
+    {
+      "epoch": 1.717485998755445,
+      "eval_loss": 1.6283434629440308,
+      "eval_runtime": 50.596,
+      "eval_samples_per_second": 19.764,
+      "eval_steps_per_second": 0.83,
+      "step": 1380
+    },
+    {
+      "epoch": 1.7187305538270068,
+      "grad_norm": 3.2046546936035156,
+      "learning_rate": 2.407749077490775e-06,
+      "loss": 1.4422,
+      "step": 1381
+    },
+    {
+      "epoch": 1.7199751088985686,
+      "grad_norm": 3.0105698108673096,
+      "learning_rate": 2.4054428044280444e-06,
+      "loss": 1.4494,
+      "step": 1382
+    },
+    {
+      "epoch": 1.7212196639701307,
+      "grad_norm": 2.9917800426483154,
+      "learning_rate": 2.403136531365314e-06,
+      "loss": 1.4407,
+      "step": 1383
+    },
+    {
+      "epoch": 1.7224642190416926,
+      "grad_norm": 2.844325065612793,
+      "learning_rate": 2.4008302583025833e-06,
+      "loss": 1.4067,
+      "step": 1384
+    },
+    {
+      "epoch": 1.7237087741132546,
+      "grad_norm": 2.919576406478882,
+      "learning_rate": 2.3985239852398527e-06,
+      "loss": 1.4352,
+      "step": 1385
+    },
+    {
+      "epoch": 1.7249533291848165,
+      "grad_norm": 2.8357744216918945,
+      "learning_rate": 2.3962177121771217e-06,
+      "loss": 1.3978,
+      "step": 1386
+    },
+    {
+      "epoch": 1.7261978842563783,
+      "grad_norm": 2.82425856590271,
+      "learning_rate": 2.3939114391143915e-06,
+      "loss": 1.4439,
+      "step": 1387
+    },
+    {
+      "epoch": 1.7274424393279402,
+      "grad_norm": 2.9155638217926025,
+      "learning_rate": 2.3916051660516605e-06,
+      "loss": 1.4326,
+      "step": 1388
+    },
+    {
+      "epoch": 1.728686994399502,
+      "grad_norm": 2.862804651260376,
+      "learning_rate": 2.3892988929889304e-06,
+      "loss": 1.3766,
+      "step": 1389
+    },
+    {
+      "epoch": 1.729931549471064,
+      "grad_norm": 3.012230634689331,
+      "learning_rate": 2.3869926199261994e-06,
+      "loss": 1.4569,
+      "step": 1390
+    },
+    {
+      "epoch": 1.729931549471064,
+      "eval_loss": 1.6268510818481445,
+      "eval_runtime": 52.5438,
+      "eval_samples_per_second": 19.032,
+      "eval_steps_per_second": 0.799,
+      "step": 1390
+    },
+    {
+      "epoch": 1.7311761045426262,
+      "grad_norm": 2.9922425746917725,
+      "learning_rate": 2.384686346863469e-06,
+      "loss": 1.4066,
+      "step": 1391
+    },
+    {
+      "epoch": 1.732420659614188,
+      "grad_norm": 2.960145950317383,
+      "learning_rate": 2.3823800738007382e-06,
+      "loss": 1.4514,
+      "step": 1392
+    },
+    {
+      "epoch": 1.7336652146857499,
+      "grad_norm": 2.834998846054077,
+      "learning_rate": 2.3800738007380077e-06,
+      "loss": 1.4714,
+      "step": 1393
+    },
+    {
+      "epoch": 1.7349097697573117,
+      "grad_norm": 2.9646077156066895,
+      "learning_rate": 2.377767527675277e-06,
+      "loss": 1.4368,
+      "step": 1394
+    },
+    {
+      "epoch": 1.7361543248288736,
+      "grad_norm": 2.8233699798583984,
+      "learning_rate": 2.3754612546125465e-06,
+      "loss": 1.4598,
+      "step": 1395
+    },
+    {
+      "epoch": 1.7373988799004356,
+      "grad_norm": 2.895472764968872,
+      "learning_rate": 2.3731549815498155e-06,
+      "loss": 1.4551,
+      "step": 1396
+    },
+    {
+      "epoch": 1.7386434349719975,
+      "grad_norm": 2.7917752265930176,
+      "learning_rate": 2.3708487084870853e-06,
+      "loss": 1.4759,
+      "step": 1397
+    },
+    {
+      "epoch": 1.7398879900435595,
+      "grad_norm": 2.820672035217285,
+      "learning_rate": 2.3685424354243543e-06,
+      "loss": 1.3711,
+      "step": 1398
+    },
+    {
+      "epoch": 1.7411325451151214,
+      "grad_norm": 2.804952621459961,
+      "learning_rate": 2.3662361623616238e-06,
+      "loss": 1.4227,
+      "step": 1399
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "grad_norm": 2.981553316116333,
+      "learning_rate": 2.363929889298893e-06,
+      "loss": 1.4639,
+      "step": 1400
+    },
+    {
+      "epoch": 1.7423771001866832,
+      "eval_loss": 1.6229957342147827,
+      "eval_runtime": 49.1507,
+      "eval_samples_per_second": 20.346,
+      "eval_steps_per_second": 0.855,
+      "step": 1400
+    },
+    {
+      "epoch": 1.743621655258245,
+      "grad_norm": 2.8649392127990723,
+      "learning_rate": 2.3616236162361626e-06,
+      "loss": 1.4283,
+      "step": 1401
+    },
+    {
+      "epoch": 1.744866210329807,
+      "grad_norm": 2.9438977241516113,
+      "learning_rate": 2.359317343173432e-06,
+      "loss": 1.444,
+      "step": 1402
+    },
+    {
+      "epoch": 1.746110765401369,
+      "grad_norm": 2.9270176887512207,
+      "learning_rate": 2.3570110701107015e-06,
+      "loss": 1.4044,
+      "step": 1403
+    },
+    {
+      "epoch": 1.747355320472931,
+      "grad_norm": 2.791527271270752,
+      "learning_rate": 2.3547047970479705e-06,
+      "loss": 1.4195,
+      "step": 1404
+    },
+    {
+      "epoch": 1.748599875544493,
+      "grad_norm": 2.8519608974456787,
+      "learning_rate": 2.3523985239852403e-06,
+      "loss": 1.4031,
+      "step": 1405
+    },
+    {
+      "epoch": 1.7498444306160548,
+      "grad_norm": 2.90872859954834,
+      "learning_rate": 2.3500922509225093e-06,
+      "loss": 1.4616,
+      "step": 1406
+    },
+    {
+      "epoch": 1.7510889856876166,
+      "grad_norm": 2.9190874099731445,
+      "learning_rate": 2.3477859778597787e-06,
+      "loss": 1.3917,
+      "step": 1407
+    },
+    {
+      "epoch": 1.7523335407591785,
+      "grad_norm": 2.868323802947998,
+      "learning_rate": 2.345479704797048e-06,
+      "loss": 1.4053,
+      "step": 1408
+    },
+    {
+      "epoch": 1.7535780958307405,
+      "grad_norm": 2.8638579845428467,
+      "learning_rate": 2.3431734317343176e-06,
+      "loss": 1.4042,
+      "step": 1409
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "grad_norm": 3.1759817600250244,
+      "learning_rate": 2.340867158671587e-06,
+      "loss": 1.4937,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7548226509023024,
+      "eval_loss": 1.6252617835998535,
+      "eval_runtime": 44.3041,
+      "eval_samples_per_second": 22.571,
+      "eval_steps_per_second": 0.948,
+      "step": 1410
+    },
+    {
+      "epoch": 1.7560672059738645,
+      "grad_norm": 2.8265459537506104,
+      "learning_rate": 2.3385608856088564e-06,
+      "loss": 1.406,
+      "step": 1411
+    },
+    {
+      "epoch": 1.7573117610454263,
+      "grad_norm": 2.856229305267334,
+      "learning_rate": 2.336254612546126e-06,
+      "loss": 1.4395,
+      "step": 1412
+    },
+    {
+      "epoch": 1.7585563161169881,
+      "grad_norm": 2.9112775325775146,
+      "learning_rate": 2.333948339483395e-06,
+      "loss": 1.427,
+      "step": 1413
+    },
+    {
+      "epoch": 1.75980087118855,
+      "grad_norm": 3.098846435546875,
+      "learning_rate": 2.3316420664206643e-06,
+      "loss": 1.4222,
+      "step": 1414
+    },
+    {
+      "epoch": 1.7610454262601118,
+      "grad_norm": 2.758100986480713,
+      "learning_rate": 2.3293357933579337e-06,
+      "loss": 1.3844,
+      "step": 1415
+    },
+    {
+      "epoch": 1.762289981331674,
+      "grad_norm": 2.8422274589538574,
+      "learning_rate": 2.327029520295203e-06,
+      "loss": 1.4033,
+      "step": 1416
+    },
+    {
+      "epoch": 1.763534536403236,
+      "grad_norm": 2.9257912635803223,
+      "learning_rate": 2.3247232472324725e-06,
+      "loss": 1.3961,
+      "step": 1417
+    },
+    {
+      "epoch": 1.7647790914747978,
+      "grad_norm": 2.9431657791137695,
+      "learning_rate": 2.322416974169742e-06,
+      "loss": 1.4451,
+      "step": 1418
+    },
+    {
+      "epoch": 1.7660236465463597,
+      "grad_norm": 2.8598814010620117,
+      "learning_rate": 2.3201107011070114e-06,
+      "loss": 1.3816,
+      "step": 1419
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "grad_norm": 2.9868006706237793,
+      "learning_rate": 2.317804428044281e-06,
+      "loss": 1.4037,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7672682016179215,
+      "eval_loss": 1.6231273412704468,
+      "eval_runtime": 48.8067,
+      "eval_samples_per_second": 20.489,
+      "eval_steps_per_second": 0.861,
+      "step": 1420
+    },
+    {
+      "epoch": 1.7685127566894834,
+      "grad_norm": 2.888206720352173,
+      "learning_rate": 2.31549815498155e-06,
+      "loss": 1.4087,
+      "step": 1421
+    },
+    {
+      "epoch": 1.7697573117610454,
+      "grad_norm": 2.9240078926086426,
+      "learning_rate": 2.3131918819188192e-06,
+      "loss": 1.3909,
+      "step": 1422
+    },
+    {
+      "epoch": 1.7710018668326073,
+      "grad_norm": 3.0401012897491455,
+      "learning_rate": 2.3108856088560886e-06,
+      "loss": 1.423,
+      "step": 1423
+    },
+    {
+      "epoch": 1.7722464219041694,
+      "grad_norm": 2.9484200477600098,
+      "learning_rate": 2.308579335793358e-06,
+      "loss": 1.449,
+      "step": 1424
+    },
+    {
+      "epoch": 1.7734909769757312,
+      "grad_norm": 2.863267660140991,
+      "learning_rate": 2.3062730627306275e-06,
+      "loss": 1.4215,
+      "step": 1425
+    },
+    {
+      "epoch": 1.774735532047293,
+      "grad_norm": 2.9615612030029297,
+      "learning_rate": 2.303966789667897e-06,
+      "loss": 1.4391,
+      "step": 1426
+    },
+    {
+      "epoch": 1.775980087118855,
+      "grad_norm": 2.7958569526672363,
+      "learning_rate": 2.3016605166051663e-06,
+      "loss": 1.3662,
+      "step": 1427
+    },
+    {
+      "epoch": 1.777224642190417,
+      "grad_norm": 2.898787021636963,
+      "learning_rate": 2.2993542435424358e-06,
+      "loss": 1.362,
+      "step": 1428
+    },
+    {
+      "epoch": 1.7784691972619788,
+      "grad_norm": 2.883049726486206,
+      "learning_rate": 2.2970479704797048e-06,
+      "loss": 1.4339,
+      "step": 1429
+    },
+    {
+      "epoch": 1.779713752333541,
+      "grad_norm": 3.0204827785491943,
+      "learning_rate": 2.294741697416974e-06,
+      "loss": 1.5174,
+      "step": 1430
+    },
+    {
+      "epoch": 1.779713752333541,
+      "eval_loss": 1.6246228218078613,
+      "eval_runtime": 47.5611,
+      "eval_samples_per_second": 21.026,
+      "eval_steps_per_second": 0.883,
+      "step": 1430
+    },
+    {
+      "epoch": 1.7809583074051027,
+      "grad_norm": 2.9354310035705566,
+      "learning_rate": 2.2924354243542436e-06,
+      "loss": 1.4115,
+      "step": 1431
+    },
+    {
+      "epoch": 1.7822028624766646,
+      "grad_norm": 2.9498608112335205,
+      "learning_rate": 2.290129151291513e-06,
+      "loss": 1.4058,
+      "step": 1432
+    },
+    {
+      "epoch": 1.7834474175482264,
+      "grad_norm": 2.9706921577453613,
+      "learning_rate": 2.2878228782287825e-06,
+      "loss": 1.426,
+      "step": 1433
+    },
+    {
+      "epoch": 1.7846919726197883,
+      "grad_norm": 2.8945024013519287,
+      "learning_rate": 2.285516605166052e-06,
+      "loss": 1.4011,
+      "step": 1434
+    },
+    {
+      "epoch": 1.7859365276913504,
+      "grad_norm": 2.8987081050872803,
+      "learning_rate": 2.2832103321033213e-06,
+      "loss": 1.4163,
+      "step": 1435
+    },
+    {
+      "epoch": 1.7871810827629122,
+      "grad_norm": 2.9797592163085938,
+      "learning_rate": 2.2809040590405907e-06,
+      "loss": 1.4367,
+      "step": 1436
+    },
+    {
+      "epoch": 1.7884256378344743,
+      "grad_norm": 2.8872063159942627,
+      "learning_rate": 2.2785977859778597e-06,
+      "loss": 1.4226,
+      "step": 1437
+    },
+    {
+      "epoch": 1.7896701929060361,
+      "grad_norm": 2.942655563354492,
+      "learning_rate": 2.2762915129151296e-06,
+      "loss": 1.4132,
+      "step": 1438
+    },
+    {
+      "epoch": 1.790914747977598,
+      "grad_norm": 3.027392625808716,
+      "learning_rate": 2.2739852398523986e-06,
+      "loss": 1.4148,
+      "step": 1439
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "grad_norm": 2.9475181102752686,
+      "learning_rate": 2.271678966789668e-06,
+      "loss": 1.4585,
+      "step": 1440
+    },
+    {
+      "epoch": 1.7921593030491598,
+      "eval_loss": 1.6234897375106812,
+      "eval_runtime": 53.3737,
+      "eval_samples_per_second": 18.736,
+      "eval_steps_per_second": 0.787,
+      "step": 1440
+    },
+    {
+      "epoch": 1.793403858120722,
+      "grad_norm": 2.846304416656494,
+      "learning_rate": 2.2693726937269374e-06,
+      "loss": 1.3667,
+      "step": 1441
+    },
+    {
+      "epoch": 1.7946484131922837,
+      "grad_norm": 2.8925700187683105,
+      "learning_rate": 2.267066420664207e-06,
+      "loss": 1.4318,
+      "step": 1442
+    },
+    {
+      "epoch": 1.7958929682638458,
+      "grad_norm": 2.884025812149048,
+      "learning_rate": 2.2647601476014763e-06,
+      "loss": 1.4395,
+      "step": 1443
+    },
+    {
+      "epoch": 1.7971375233354077,
+      "grad_norm": 2.9763753414154053,
+      "learning_rate": 2.2624538745387457e-06,
+      "loss": 1.3742,
+      "step": 1444
+    },
+    {
+      "epoch": 1.7983820784069695,
+      "grad_norm": 3.0834898948669434,
+      "learning_rate": 2.2601476014760147e-06,
+      "loss": 1.4651,
+      "step": 1445
+    },
+    {
+      "epoch": 1.7996266334785314,
+      "grad_norm": 3.0673978328704834,
+      "learning_rate": 2.2578413284132845e-06,
+      "loss": 1.475,
+      "step": 1446
+    },
+    {
+      "epoch": 1.8008711885500932,
+      "grad_norm": 2.931521415710449,
+      "learning_rate": 2.2555350553505535e-06,
+      "loss": 1.4506,
+      "step": 1447
+    },
+    {
+      "epoch": 1.8021157436216553,
+      "grad_norm": 2.953301429748535,
+      "learning_rate": 2.253228782287823e-06,
+      "loss": 1.4224,
+      "step": 1448
+    },
+    {
+      "epoch": 1.8033602986932171,
+      "grad_norm": 3.0210883617401123,
+      "learning_rate": 2.2509225092250924e-06,
+      "loss": 1.4387,
+      "step": 1449
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "grad_norm": 2.975741386413574,
+      "learning_rate": 2.248616236162362e-06,
+      "loss": 1.4318,
+      "step": 1450
+    },
+    {
+      "epoch": 1.8046048537647792,
+      "eval_loss": 1.618930697441101,
+      "eval_runtime": 49.6692,
+      "eval_samples_per_second": 20.133,
+      "eval_steps_per_second": 0.846,
+      "step": 1450
+    },
+    {
+      "epoch": 1.805849408836341,
+      "grad_norm": 2.9083876609802246,
+      "learning_rate": 2.2463099630996312e-06,
+      "loss": 1.3837,
+      "step": 1451
+    },
+    {
+      "epoch": 1.807093963907903,
+      "grad_norm": 3.364248275756836,
+      "learning_rate": 2.2440036900369006e-06,
+      "loss": 1.4589,
+      "step": 1452
+    },
+    {
+      "epoch": 1.8083385189794647,
+      "grad_norm": 2.935478925704956,
+      "learning_rate": 2.2416974169741696e-06,
+      "loss": 1.4119,
+      "step": 1453
+    },
+    {
+      "epoch": 1.8095830740510268,
+      "grad_norm": 3.1742911338806152,
+      "learning_rate": 2.2393911439114395e-06,
+      "loss": 1.4203,
+      "step": 1454
+    },
+    {
+      "epoch": 1.8108276291225887,
+      "grad_norm": 2.96966290473938,
+      "learning_rate": 2.2370848708487085e-06,
+      "loss": 1.4054,
+      "step": 1455
+    },
+    {
+      "epoch": 1.8120721841941507,
+      "grad_norm": 3.0919055938720703,
+      "learning_rate": 2.2347785977859783e-06,
+      "loss": 1.4289,
+      "step": 1456
+    },
+    {
+      "epoch": 1.8133167392657126,
+      "grad_norm": 2.9061055183410645,
+      "learning_rate": 2.2324723247232473e-06,
+      "loss": 1.4958,
+      "step": 1457
+    },
+    {
+      "epoch": 1.8145612943372744,
+      "grad_norm": 2.968348503112793,
+      "learning_rate": 2.2301660516605168e-06,
+      "loss": 1.4033,
+      "step": 1458
+    },
+    {
+      "epoch": 1.8158058494088363,
+      "grad_norm": 3.0435378551483154,
+      "learning_rate": 2.227859778597786e-06,
+      "loss": 1.414,
+      "step": 1459
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "grad_norm": 2.9067916870117188,
+      "learning_rate": 2.2255535055350556e-06,
+      "loss": 1.3954,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8170504044803981,
+      "eval_loss": 1.6240814924240112,
+      "eval_runtime": 51.7164,
+      "eval_samples_per_second": 19.336,
+      "eval_steps_per_second": 0.812,
+      "step": 1460
+    },
+    {
+      "epoch": 1.8182949595519602,
+      "grad_norm": 3.156384229660034,
+      "learning_rate": 2.2232472324723246e-06,
+      "loss": 1.4658,
+      "step": 1461
+    },
+    {
+      "epoch": 1.819539514623522,
+      "grad_norm": 2.9885313510894775,
+      "learning_rate": 2.2209409594095944e-06,
+      "loss": 1.4123,
+      "step": 1462
+    },
+    {
+      "epoch": 1.820784069695084,
+      "grad_norm": 3.227780818939209,
+      "learning_rate": 2.2186346863468634e-06,
+      "loss": 1.3587,
+      "step": 1463
+    },
+    {
+      "epoch": 1.822028624766646,
+      "grad_norm": 3.0375192165374756,
+      "learning_rate": 2.2163284132841333e-06,
+      "loss": 1.4015,
+      "step": 1464
+    },
+    {
+      "epoch": 1.8232731798382078,
+      "grad_norm": 3.0200846195220947,
+      "learning_rate": 2.2140221402214023e-06,
+      "loss": 1.419,
+      "step": 1465
+    },
+    {
+      "epoch": 1.8245177349097697,
+      "grad_norm": 2.9456615447998047,
+      "learning_rate": 2.2117158671586717e-06,
+      "loss": 1.4075,
+      "step": 1466
+    },
+    {
+      "epoch": 1.8257622899813317,
+      "grad_norm": 3.3129165172576904,
+      "learning_rate": 2.209409594095941e-06,
+      "loss": 1.4241,
+      "step": 1467
+    },
+    {
+      "epoch": 1.8270068450528936,
+      "grad_norm": 3.013141393661499,
+      "learning_rate": 2.2071033210332106e-06,
+      "loss": 1.4066,
+      "step": 1468
+    },
+    {
+      "epoch": 1.8282514001244556,
+      "grad_norm": 2.8414366245269775,
+      "learning_rate": 2.20479704797048e-06,
+      "loss": 1.4009,
+      "step": 1469
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "grad_norm": 2.8920438289642334,
+      "learning_rate": 2.2024907749077494e-06,
+      "loss": 1.3916,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8294959551960175,
+      "eval_loss": 1.6220412254333496,
+      "eval_runtime": 52.6199,
+      "eval_samples_per_second": 19.004,
+      "eval_steps_per_second": 0.798,
+      "step": 1470
+    },
+    {
+      "epoch": 1.8307405102675793,
+      "grad_norm": 3.032353639602661,
+      "learning_rate": 2.2001845018450184e-06,
+      "loss": 1.402,
+      "step": 1471
+    },
+    {
+      "epoch": 1.8319850653391412,
+      "grad_norm": 2.9512081146240234,
+      "learning_rate": 2.1978782287822883e-06,
+      "loss": 1.429,
+      "step": 1472
+    },
+    {
+      "epoch": 1.833229620410703,
+      "grad_norm": 2.846081018447876,
+      "learning_rate": 2.1955719557195573e-06,
+      "loss": 1.4203,
+      "step": 1473
+    },
+    {
+      "epoch": 1.834474175482265,
+      "grad_norm": 3.093378782272339,
+      "learning_rate": 2.1932656826568267e-06,
+      "loss": 1.4307,
+      "step": 1474
+    },
+    {
+      "epoch": 1.8357187305538272,
+      "grad_norm": 2.808326005935669,
+      "learning_rate": 2.190959409594096e-06,
+      "loss": 1.3753,
+      "step": 1475
+    },
+    {
+      "epoch": 1.836963285625389,
+      "grad_norm": 3.0328383445739746,
+      "learning_rate": 2.1886531365313655e-06,
+      "loss": 1.4534,
+      "step": 1476
+    },
+    {
+      "epoch": 1.8382078406969509,
+      "grad_norm": 3.045897960662842,
+      "learning_rate": 2.186346863468635e-06,
+      "loss": 1.3259,
+      "step": 1477
+    },
+    {
+      "epoch": 1.8394523957685127,
+      "grad_norm": 3.171846628189087,
+      "learning_rate": 2.1840405904059044e-06,
+      "loss": 1.397,
+      "step": 1478
+    },
+    {
+      "epoch": 1.8406969508400746,
+      "grad_norm": 2.893061637878418,
+      "learning_rate": 2.1817343173431734e-06,
+      "loss": 1.4201,
+      "step": 1479
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "grad_norm": 3.0151467323303223,
+      "learning_rate": 2.1794280442804432e-06,
+      "loss": 1.4266,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8419415059116366,
+      "eval_loss": 1.6166906356811523,
+      "eval_runtime": 51.8266,
+      "eval_samples_per_second": 19.295,
+      "eval_steps_per_second": 0.81,
+      "step": 1480
+    },
+    {
+      "epoch": 1.8431860609831985,
+      "grad_norm": 2.942431688308716,
+      "learning_rate": 2.177121771217712e-06,
+      "loss": 1.3744,
+      "step": 1481
+    },
+    {
+      "epoch": 1.8444306160547606,
+      "grad_norm": 2.8482232093811035,
+      "learning_rate": 2.1748154981549816e-06,
+      "loss": 1.3914,
+      "step": 1482
+    },
+    {
+      "epoch": 1.8456751711263224,
+      "grad_norm": 2.9315359592437744,
+      "learning_rate": 2.172509225092251e-06,
+      "loss": 1.419,
+      "step": 1483
+    },
+    {
+      "epoch": 1.8469197261978842,
+      "grad_norm": 3.191800594329834,
+      "learning_rate": 2.1702029520295205e-06,
+      "loss": 1.3524,
+      "step": 1484
+    },
+    {
+      "epoch": 1.848164281269446,
+      "grad_norm": 2.9308040142059326,
+      "learning_rate": 2.16789667896679e-06,
+      "loss": 1.3742,
+      "step": 1485
+    },
+    {
+      "epoch": 1.849408836341008,
+      "grad_norm": 2.7678964138031006,
+      "learning_rate": 2.1655904059040593e-06,
+      "loss": 1.3786,
+      "step": 1486
+    },
+    {
+      "epoch": 1.85065339141257,
+      "grad_norm": 3.0300636291503906,
+      "learning_rate": 2.1632841328413287e-06,
+      "loss": 1.4469,
+      "step": 1487
+    },
+    {
+      "epoch": 1.851897946484132,
+      "grad_norm": 3.142781972885132,
+      "learning_rate": 2.160977859778598e-06,
+      "loss": 1.4565,
+      "step": 1488
+    },
+    {
+      "epoch": 1.853142501555694,
+      "grad_norm": 2.954916000366211,
+      "learning_rate": 2.158671586715867e-06,
+      "loss": 1.4356,
+      "step": 1489
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "grad_norm": 2.864875316619873,
+      "learning_rate": 2.1563653136531366e-06,
+      "loss": 1.4521,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8543870566272558,
+      "eval_loss": 1.6207302808761597,
+      "eval_runtime": 49.2215,
+      "eval_samples_per_second": 20.316,
+      "eval_steps_per_second": 0.853,
+      "step": 1490
+    },
+    {
+      "epoch": 1.8556316116988176,
+      "grad_norm": 2.8908467292785645,
+      "learning_rate": 2.154059040590406e-06,
+      "loss": 1.4304,
+      "step": 1491
+    },
+    {
+      "epoch": 1.8568761667703795,
+      "grad_norm": 2.9439549446105957,
+      "learning_rate": 2.1517527675276754e-06,
+      "loss": 1.398,
+      "step": 1492
+    },
+    {
+      "epoch": 1.8581207218419415,
+      "grad_norm": 2.967728853225708,
+      "learning_rate": 2.149446494464945e-06,
+      "loss": 1.5038,
+      "step": 1493
+    },
+    {
+      "epoch": 1.8593652769135034,
+      "grad_norm": 2.938123941421509,
+      "learning_rate": 2.1471402214022143e-06,
+      "loss": 1.4395,
+      "step": 1494
+    },
+    {
+      "epoch": 1.8606098319850655,
+      "grad_norm": 2.93609881401062,
+      "learning_rate": 2.1448339483394837e-06,
+      "loss": 1.3939,
+      "step": 1495
+    },
+    {
+      "epoch": 1.8618543870566273,
+      "grad_norm": 2.9093775749206543,
+      "learning_rate": 2.142527675276753e-06,
+      "loss": 1.3942,
+      "step": 1496
+    },
+    {
+      "epoch": 1.8630989421281892,
+      "grad_norm": 2.773960590362549,
+      "learning_rate": 2.140221402214022e-06,
+      "loss": 1.3886,
+      "step": 1497
+    },
+    {
+      "epoch": 1.864343497199751,
+      "grad_norm": 2.9858205318450928,
+      "learning_rate": 2.1379151291512916e-06,
+      "loss": 1.3814,
+      "step": 1498
+    },
+    {
+      "epoch": 1.8655880522713129,
+      "grad_norm": 3.1233417987823486,
+      "learning_rate": 2.135608856088561e-06,
+      "loss": 1.4313,
+      "step": 1499
+    },
+    {
+      "epoch": 1.866832607342875,
+      "grad_norm": 2.9569480419158936,
+      "learning_rate": 2.1333025830258304e-06,
+      "loss": 1.4302,
+      "step": 1500
+    },
+    {
+      "epoch": 1.866832607342875,
+      "eval_loss": 1.6130231618881226,
+      "eval_runtime": 46.2834,
+      "eval_samples_per_second": 21.606,
+      "eval_steps_per_second": 0.907,
+      "step": 1500
+    },
+    {
+      "epoch": 1.868077162414437,
+      "grad_norm": 2.926203727722168,
+      "learning_rate": 2.1309963099631e-06,
+      "loss": 1.4236,
+      "step": 1501
+    },
+    {
+      "epoch": 1.8693217174859988,
+      "grad_norm": 2.9496357440948486,
+      "learning_rate": 2.1286900369003692e-06,
+      "loss": 1.4014,
+      "step": 1502
+    },
+    {
+      "epoch": 1.8705662725575607,
+      "grad_norm": 2.9314279556274414,
+      "learning_rate": 2.1263837638376387e-06,
+      "loss": 1.4414,
+      "step": 1503
+    },
+    {
+      "epoch": 1.8718108276291225,
+      "grad_norm": 2.889460802078247,
+      "learning_rate": 2.124077490774908e-06,
+      "loss": 1.3883,
+      "step": 1504
+    },
+    {
+      "epoch": 1.8730553827006844,
+      "grad_norm": 2.9859671592712402,
+      "learning_rate": 2.1217712177121775e-06,
+      "loss": 1.4551,
+      "step": 1505
+    },
+    {
+      "epoch": 1.8742999377722465,
+      "grad_norm": 2.9330570697784424,
+      "learning_rate": 2.1194649446494465e-06,
+      "loss": 1.3895,
+      "step": 1506
+    },
+    {
+      "epoch": 1.8755444928438083,
+      "grad_norm": 2.9062387943267822,
+      "learning_rate": 2.117158671586716e-06,
+      "loss": 1.4492,
+      "step": 1507
+    },
+    {
+      "epoch": 1.8767890479153704,
+      "grad_norm": 3.0337791442871094,
+      "learning_rate": 2.1148523985239854e-06,
+      "loss": 1.4682,
+      "step": 1508
+    },
+    {
+      "epoch": 1.8780336029869322,
+      "grad_norm": 3.0448431968688965,
+      "learning_rate": 2.1125461254612548e-06,
+      "loss": 1.4836,
+      "step": 1509
+    },
+    {
+      "epoch": 1.879278158058494,
+      "grad_norm": 2.912764310836792,
+      "learning_rate": 2.110239852398524e-06,
+      "loss": 1.4644,
+      "step": 1510
+    },
+    {
+      "epoch": 1.879278158058494,
+      "eval_loss": 1.6124104261398315,
+      "eval_runtime": 41.9713,
+      "eval_samples_per_second": 23.826,
+      "eval_steps_per_second": 1.001,
+      "step": 1510
+    },
+    {
+      "epoch": 1.880522713130056,
+      "grad_norm": 3.0399203300476074,
+      "learning_rate": 2.1079335793357936e-06,
+      "loss": 1.4391,
+      "step": 1511
+    },
+    {
+      "epoch": 1.8817672682016178,
+      "grad_norm": 2.9852235317230225,
+      "learning_rate": 2.1056273062730626e-06,
+      "loss": 1.346,
+      "step": 1512
+    },
+    {
+      "epoch": 1.8830118232731798,
+      "grad_norm": 2.904437780380249,
+      "learning_rate": 2.1033210332103325e-06,
+      "loss": 1.4523,
+      "step": 1513
+    },
+    {
+      "epoch": 1.884256378344742,
+      "grad_norm": 2.8988049030303955,
+      "learning_rate": 2.1010147601476015e-06,
+      "loss": 1.4001,
+      "step": 1514
+    },
+    {
+      "epoch": 1.8855009334163038,
+      "grad_norm": 2.9890260696411133,
+      "learning_rate": 2.098708487084871e-06,
+      "loss": 1.4074,
+      "step": 1515
+    },
+    {
+      "epoch": 1.8867454884878656,
+      "grad_norm": 3.0486819744110107,
+      "learning_rate": 2.0964022140221403e-06,
+      "loss": 1.461,
+      "step": 1516
+    },
+    {
+      "epoch": 1.8879900435594275,
+      "grad_norm": 2.8746016025543213,
+      "learning_rate": 2.0940959409594097e-06,
+      "loss": 1.4013,
+      "step": 1517
+    },
+    {
+      "epoch": 1.8892345986309893,
+      "grad_norm": 2.979480266571045,
+      "learning_rate": 2.091789667896679e-06,
+      "loss": 1.4567,
+      "step": 1518
+    },
+    {
+      "epoch": 1.8904791537025514,
+      "grad_norm": 2.8351497650146484,
+      "learning_rate": 2.0894833948339486e-06,
+      "loss": 1.3655,
+      "step": 1519
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "grad_norm": 2.9061646461486816,
+      "learning_rate": 2.0871771217712176e-06,
+      "loss": 1.4165,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8917237087741132,
+      "eval_loss": 1.6172568798065186,
+      "eval_runtime": 42.998,
+      "eval_samples_per_second": 23.257,
+      "eval_steps_per_second": 0.977,
+      "step": 1520
+    },
+    {
+      "epoch": 1.8929682638456753,
+      "grad_norm": 2.8751838207244873,
+      "learning_rate": 2.0848708487084874e-06,
+      "loss": 1.4133,
+      "step": 1521
+    },
+    {
+      "epoch": 1.8942128189172371,
+      "grad_norm": 2.7513587474823,
+      "learning_rate": 2.0825645756457564e-06,
+      "loss": 1.3734,
+      "step": 1522
+    },
+    {
+      "epoch": 1.895457373988799,
+      "grad_norm": 2.9244418144226074,
+      "learning_rate": 2.0802583025830263e-06,
+      "loss": 1.4181,
+      "step": 1523
+    },
+    {
+      "epoch": 1.8967019290603608,
+      "grad_norm": 2.877495765686035,
+      "learning_rate": 2.0779520295202953e-06,
+      "loss": 1.4009,
+      "step": 1524
+    },
+    {
+      "epoch": 1.8979464841319227,
+      "grad_norm": 2.9316184520721436,
+      "learning_rate": 2.0756457564575647e-06,
+      "loss": 1.3796,
+      "step": 1525
+    },
+    {
+      "epoch": 1.8991910392034848,
+      "grad_norm": 3.0403976440429688,
+      "learning_rate": 2.073339483394834e-06,
+      "loss": 1.4101,
+      "step": 1526
+    },
+    {
+      "epoch": 1.9004355942750468,
+      "grad_norm": 3.0020930767059326,
+      "learning_rate": 2.0710332103321035e-06,
+      "loss": 1.4166,
+      "step": 1527
+    },
+    {
+      "epoch": 1.9016801493466087,
+      "grad_norm": 3.0937304496765137,
+      "learning_rate": 2.0687269372693725e-06,
+      "loss": 1.4599,
+      "step": 1528
+    },
+    {
+      "epoch": 1.9029247044181705,
+      "grad_norm": 2.901540517807007,
+      "learning_rate": 2.0664206642066424e-06,
+      "loss": 1.3642,
+      "step": 1529
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "grad_norm": 3.0517938137054443,
+      "learning_rate": 2.0641143911439114e-06,
+      "loss": 1.451,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9041692594897324,
+      "eval_loss": 1.6178728342056274,
+      "eval_runtime": 43.9067,
+      "eval_samples_per_second": 22.776,
+      "eval_steps_per_second": 0.957,
+      "step": 1530
+    },
+    {
+      "epoch": 1.9054138145612942,
+      "grad_norm": 3.12484073638916,
+      "learning_rate": 2.0618081180811812e-06,
+      "loss": 1.4522,
+      "step": 1531
+    },
+    {
+      "epoch": 1.9066583696328563,
+      "grad_norm": 2.9447414875030518,
+      "learning_rate": 2.0595018450184502e-06,
+      "loss": 1.4154,
+      "step": 1532
+    },
+    {
+      "epoch": 1.9079029247044181,
+      "grad_norm": 2.909738779067993,
+      "learning_rate": 2.0571955719557197e-06,
+      "loss": 1.4133,
+      "step": 1533
+    },
+    {
+      "epoch": 1.9091474797759802,
+      "grad_norm": 3.1522421836853027,
+      "learning_rate": 2.054889298892989e-06,
+      "loss": 1.4777,
+      "step": 1534
+    },
+    {
+      "epoch": 1.910392034847542,
+      "grad_norm": 3.041447639465332,
+      "learning_rate": 2.0525830258302585e-06,
+      "loss": 1.4104,
+      "step": 1535
+    },
+    {
+      "epoch": 1.911636589919104,
+      "grad_norm": 3.01129412651062,
+      "learning_rate": 2.050276752767528e-06,
+      "loss": 1.3852,
+      "step": 1536
+    },
+    {
+      "epoch": 1.9128811449906657,
+      "grad_norm": 3.0697543621063232,
+      "learning_rate": 2.0479704797047974e-06,
+      "loss": 1.4389,
+      "step": 1537
+    },
+    {
+      "epoch": 1.9141257000622276,
+      "grad_norm": 2.9391236305236816,
+      "learning_rate": 2.0456642066420664e-06,
+      "loss": 1.4239,
+      "step": 1538
+    },
+    {
+      "epoch": 1.9153702551337897,
+      "grad_norm": 2.978236675262451,
+      "learning_rate": 2.043357933579336e-06,
+      "loss": 1.462,
+      "step": 1539
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "grad_norm": 2.8124167919158936,
+      "learning_rate": 2.041051660516605e-06,
+      "loss": 1.3186,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9166148102053517,
+      "eval_loss": 1.6120468378067017,
+      "eval_runtime": 43.3787,
+      "eval_samples_per_second": 23.053,
+      "eval_steps_per_second": 0.968,
+      "step": 1540
+    },
+    {
+      "epoch": 1.9178593652769136,
+      "grad_norm": 3.029939889907837,
+      "learning_rate": 2.0387453874538746e-06,
+      "loss": 1.4481,
+      "step": 1541
+    },
+    {
+      "epoch": 1.9191039203484754,
+      "grad_norm": 2.8749356269836426,
+      "learning_rate": 2.036439114391144e-06,
+      "loss": 1.3735,
+      "step": 1542
+    },
+    {
+      "epoch": 1.9203484754200373,
+      "grad_norm": 2.9181101322174072,
+      "learning_rate": 2.0341328413284135e-06,
+      "loss": 1.4085,
+      "step": 1543
+    },
+    {
+      "epoch": 1.9215930304915991,
+      "grad_norm": 2.936319589614868,
+      "learning_rate": 2.031826568265683e-06,
+      "loss": 1.3827,
+      "step": 1544
+    },
+    {
+      "epoch": 1.9228375855631612,
+      "grad_norm": 2.9286844730377197,
+      "learning_rate": 2.0295202952029523e-06,
+      "loss": 1.4563,
+      "step": 1545
+    },
+    {
+      "epoch": 1.924082140634723,
+      "grad_norm": 2.790752649307251,
+      "learning_rate": 2.0272140221402213e-06,
+      "loss": 1.3935,
+      "step": 1546
+    },
+    {
+      "epoch": 1.9253266957062851,
+      "grad_norm": 2.878164529800415,
+      "learning_rate": 2.024907749077491e-06,
+      "loss": 1.4159,
+      "step": 1547
+    },
+    {
+      "epoch": 1.926571250777847,
+      "grad_norm": 3.0583767890930176,
+      "learning_rate": 2.02260147601476e-06,
+      "loss": 1.4036,
+      "step": 1548
+    },
+    {
+      "epoch": 1.9278158058494088,
+      "grad_norm": 2.943686008453369,
+      "learning_rate": 2.02029520295203e-06,
+      "loss": 1.4178,
+      "step": 1549
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "grad_norm": 2.983553886413574,
+      "learning_rate": 2.017988929889299e-06,
+      "loss": 1.3959,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9290603609209707,
+      "eval_loss": 1.610864520072937,
+      "eval_runtime": 44.5262,
+      "eval_samples_per_second": 22.459,
+      "eval_steps_per_second": 0.943,
+      "step": 1550
+    },
+    {
+      "epoch": 1.9303049159925325,
+      "grad_norm": 3.0323703289031982,
+      "learning_rate": 2.0156826568265684e-06,
+      "loss": 1.4362,
+      "step": 1551
+    },
+    {
+      "epoch": 1.9315494710640946,
+      "grad_norm": 3.036627769470215,
+      "learning_rate": 2.013376383763838e-06,
+      "loss": 1.4132,
+      "step": 1552
+    },
+    {
+      "epoch": 1.9327940261356567,
+      "grad_norm": 2.901071310043335,
+      "learning_rate": 2.0110701107011073e-06,
+      "loss": 1.3846,
+      "step": 1553
+    },
+    {
+      "epoch": 1.9340385812072185,
+      "grad_norm": 3.002803325653076,
+      "learning_rate": 2.0087638376383767e-06,
+      "loss": 1.4402,
+      "step": 1554
+    },
+    {
+      "epoch": 1.9352831362787803,
+      "grad_norm": 2.969228982925415,
+      "learning_rate": 2.006457564575646e-06,
+      "loss": 1.4387,
+      "step": 1555
+    },
+    {
+      "epoch": 1.9365276913503422,
+      "grad_norm": 2.9773998260498047,
+      "learning_rate": 2.004151291512915e-06,
+      "loss": 1.4149,
+      "step": 1556
+    },
+    {
+      "epoch": 1.937772246421904,
+      "grad_norm": 2.831655502319336,
+      "learning_rate": 2.0018450184501845e-06,
+      "loss": 1.3753,
+      "step": 1557
+    },
+    {
+      "epoch": 1.9390168014934661,
+      "grad_norm": 2.835994005203247,
+      "learning_rate": 1.999538745387454e-06,
+      "loss": 1.345,
+      "step": 1558
+    },
+    {
+      "epoch": 1.940261356565028,
+      "grad_norm": 3.0328211784362793,
+      "learning_rate": 1.9972324723247234e-06,
+      "loss": 1.3473,
+      "step": 1559
+    },
+    {
+      "epoch": 1.94150591163659,
+      "grad_norm": 2.9441378116607666,
+      "learning_rate": 1.994926199261993e-06,
+      "loss": 1.3913,
+      "step": 1560
+    },
+    {
+      "epoch": 1.94150591163659,
+      "eval_loss": 1.6193854808807373,
+      "eval_runtime": 44.6194,
+      "eval_samples_per_second": 22.412,
+      "eval_steps_per_second": 0.941,
+      "step": 1560
+    },
+    {
+      "epoch": 1.9427504667081519,
+      "grad_norm": 2.905486822128296,
+      "learning_rate": 1.9926199261992622e-06,
+      "loss": 1.4043,
+      "step": 1561
+    },
+    {
+      "epoch": 1.9439950217797137,
+      "grad_norm": 2.7758026123046875,
+      "learning_rate": 1.9903136531365317e-06,
+      "loss": 1.3974,
+      "step": 1562
+    },
+    {
+      "epoch": 1.9452395768512756,
+      "grad_norm": 2.7883355617523193,
+      "learning_rate": 1.988007380073801e-06,
+      "loss": 1.4246,
+      "step": 1563
+    },
+    {
+      "epoch": 1.9464841319228374,
+      "grad_norm": 2.8322808742523193,
+      "learning_rate": 1.98570110701107e-06,
+      "loss": 1.4002,
+      "step": 1564
+    },
+    {
+      "epoch": 1.9477286869943995,
+      "grad_norm": 2.948850631713867,
+      "learning_rate": 1.9833948339483395e-06,
+      "loss": 1.3866,
+      "step": 1565
+    },
+    {
+      "epoch": 1.9489732420659616,
+      "grad_norm": 2.9772586822509766,
+      "learning_rate": 1.981088560885609e-06,
+      "loss": 1.4473,
+      "step": 1566
+    },
+    {
+      "epoch": 1.9502177971375234,
+      "grad_norm": 2.9183740615844727,
+      "learning_rate": 1.9787822878228783e-06,
+      "loss": 1.3943,
+      "step": 1567
+    },
+    {
+      "epoch": 1.9514623522090853,
+      "grad_norm": 3.10422682762146,
+      "learning_rate": 1.9764760147601478e-06,
+      "loss": 1.465,
+      "step": 1568
+    },
+    {
+      "epoch": 1.952706907280647,
+      "grad_norm": 3.001533031463623,
+      "learning_rate": 1.974169741697417e-06,
+      "loss": 1.4133,
+      "step": 1569
+    },
+    {
+      "epoch": 1.953951462352209,
+      "grad_norm": 2.9521870613098145,
+      "learning_rate": 1.9718634686346866e-06,
+      "loss": 1.4652,
+      "step": 1570
+    },
+    {
+      "epoch": 1.953951462352209,
+      "eval_loss": 1.617977499961853,
+      "eval_runtime": 45.7189,
+      "eval_samples_per_second": 21.873,
+      "eval_steps_per_second": 0.919,
+      "step": 1570
+    },
+    {
+      "epoch": 1.955196017423771,
+      "grad_norm": 2.9504826068878174,
+      "learning_rate": 1.969557195571956e-06,
+      "loss": 1.4117,
+      "step": 1571
+    },
+    {
+      "epoch": 1.9564405724953329,
+      "grad_norm": 2.9383559226989746,
+      "learning_rate": 1.9672509225092255e-06,
+      "loss": 1.3439,
+      "step": 1572
+    },
+    {
+      "epoch": 1.957685127566895,
+      "grad_norm": 2.850644588470459,
+      "learning_rate": 1.9649446494464945e-06,
+      "loss": 1.3521,
+      "step": 1573
+    },
+    {
+      "epoch": 1.9589296826384568,
+      "grad_norm": 2.8787410259246826,
+      "learning_rate": 1.962638376383764e-06,
+      "loss": 1.4268,
+      "step": 1574
+    },
+    {
+      "epoch": 1.9601742377100186,
+      "grad_norm": 2.966080665588379,
+      "learning_rate": 1.9603321033210333e-06,
+      "loss": 1.3367,
+      "step": 1575
+    },
+    {
+      "epoch": 1.9614187927815805,
+      "grad_norm": 2.858222007751465,
+      "learning_rate": 1.9580258302583027e-06,
+      "loss": 1.3842,
+      "step": 1576
+    },
+    {
+      "epoch": 1.9626633478531423,
+      "grad_norm": 2.924525737762451,
+      "learning_rate": 1.955719557195572e-06,
+      "loss": 1.4561,
+      "step": 1577
+    },
+    {
+      "epoch": 1.9639079029247044,
+      "grad_norm": 2.9281182289123535,
+      "learning_rate": 1.9534132841328416e-06,
+      "loss": 1.4179,
+      "step": 1578
+    },
+    {
+      "epoch": 1.9651524579962665,
+      "grad_norm": 2.982023000717163,
+      "learning_rate": 1.951107011070111e-06,
+      "loss": 1.4413,
+      "step": 1579
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "grad_norm": 2.7912652492523193,
+      "learning_rate": 1.9488007380073804e-06,
+      "loss": 1.3577,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9663970130678283,
+      "eval_loss": 1.6062211990356445,
+      "eval_runtime": 46.7931,
+      "eval_samples_per_second": 21.371,
+      "eval_steps_per_second": 0.898,
+      "step": 1580
+    },
+    {
+      "epoch": 1.9676415681393902,
+      "grad_norm": 2.9585583209991455,
+      "learning_rate": 1.9464944649446494e-06,
+      "loss": 1.4084,
+      "step": 1581
+    },
+    {
+      "epoch": 1.968886123210952,
+      "grad_norm": 2.879201889038086,
+      "learning_rate": 1.944188191881919e-06,
+      "loss": 1.4164,
+      "step": 1582
+    },
+    {
+      "epoch": 1.9701306782825139,
+      "grad_norm": 2.9510116577148438,
+      "learning_rate": 1.9418819188191883e-06,
+      "loss": 1.4211,
+      "step": 1583
+    },
+    {
+      "epoch": 1.971375233354076,
+      "grad_norm": 2.973844528198242,
+      "learning_rate": 1.9395756457564577e-06,
+      "loss": 1.4358,
+      "step": 1584
+    },
+    {
+      "epoch": 1.9726197884256378,
+      "grad_norm": 2.939131736755371,
+      "learning_rate": 1.937269372693727e-06,
+      "loss": 1.407,
+      "step": 1585
+    },
+    {
+      "epoch": 1.9738643434971999,
+      "grad_norm": 2.956646680831909,
+      "learning_rate": 1.9349630996309965e-06,
+      "loss": 1.3872,
+      "step": 1586
+    },
+    {
+      "epoch": 1.9751088985687617,
+      "grad_norm": 3.070354461669922,
+      "learning_rate": 1.932656826568266e-06,
+      "loss": 1.412,
+      "step": 1587
+    },
+    {
+      "epoch": 1.9763534536403236,
+      "grad_norm": 2.8143749237060547,
+      "learning_rate": 1.9303505535055354e-06,
+      "loss": 1.3522,
+      "step": 1588
+    },
+    {
+      "epoch": 1.9775980087118854,
+      "grad_norm": 2.8610002994537354,
+      "learning_rate": 1.9280442804428044e-06,
+      "loss": 1.3988,
+      "step": 1589
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "grad_norm": 2.951023817062378,
+      "learning_rate": 1.925738007380074e-06,
+      "loss": 1.3855,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9788425637834475,
+      "eval_loss": 1.613527536392212,
+      "eval_runtime": 46.5242,
+      "eval_samples_per_second": 21.494,
+      "eval_steps_per_second": 0.903,
+      "step": 1590
+    },
+    {
+      "epoch": 1.9800871188550093,
+      "grad_norm": 2.9333510398864746,
+      "learning_rate": 1.9234317343173432e-06,
+      "loss": 1.4518,
+      "step": 1591
+    },
+    {
+      "epoch": 1.9813316739265714,
+      "grad_norm": 2.847015619277954,
+      "learning_rate": 1.9211254612546126e-06,
+      "loss": 1.3999,
+      "step": 1592
+    },
+    {
+      "epoch": 1.9825762289981332,
+      "grad_norm": 2.969320058822632,
+      "learning_rate": 1.918819188191882e-06,
+      "loss": 1.4241,
+      "step": 1593
+    },
+    {
+      "epoch": 1.983820784069695,
+      "grad_norm": 2.862797498703003,
+      "learning_rate": 1.9165129151291515e-06,
+      "loss": 1.3929,
+      "step": 1594
+    },
+    {
+      "epoch": 1.985065339141257,
+      "grad_norm": 2.90384840965271,
+      "learning_rate": 1.914206642066421e-06,
+      "loss": 1.4127,
+      "step": 1595
+    },
+    {
+      "epoch": 1.9863098942128188,
+      "grad_norm": 2.8059256076812744,
+      "learning_rate": 1.9119003690036903e-06,
+      "loss": 1.4155,
+      "step": 1596
+    },
+    {
+      "epoch": 1.9875544492843809,
+      "grad_norm": 2.9068455696105957,
+      "learning_rate": 1.9095940959409593e-06,
+      "loss": 1.3924,
+      "step": 1597
+    },
+    {
+      "epoch": 1.9887990043559427,
+      "grad_norm": 2.903542995452881,
+      "learning_rate": 1.907287822878229e-06,
+      "loss": 1.4039,
+      "step": 1598
+    },
+    {
+      "epoch": 1.9900435594275048,
+      "grad_norm": 2.8566174507141113,
+      "learning_rate": 1.9049815498154984e-06,
+      "loss": 1.4276,
+      "step": 1599
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "grad_norm": 2.8891515731811523,
+      "learning_rate": 1.9026752767527676e-06,
+      "loss": 1.4555,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9912881144990666,
+      "eval_loss": 1.6086686849594116,
+      "eval_runtime": 46.2785,
+      "eval_samples_per_second": 21.608,
+      "eval_steps_per_second": 0.908,
+      "step": 1600
+    },
+    {
+      "epoch": 1.9925326695706285,
+      "grad_norm": 2.933851957321167,
+      "learning_rate": 1.900369003690037e-06,
+      "loss": 1.3881,
+      "step": 1601
+    },
+    {
+      "epoch": 1.9937772246421903,
+      "grad_norm": 2.8393521308898926,
+      "learning_rate": 1.8980627306273065e-06,
+      "loss": 1.3959,
+      "step": 1602
+    },
+    {
+      "epoch": 1.9950217797137524,
+      "grad_norm": 2.839740753173828,
+      "learning_rate": 1.8957564575645759e-06,
+      "loss": 1.384,
+      "step": 1603
+    },
+    {
+      "epoch": 1.9962663347853142,
+      "grad_norm": 2.7918097972869873,
+      "learning_rate": 1.8934501845018453e-06,
+      "loss": 1.4173,
+      "step": 1604
+    },
+    {
+      "epoch": 1.9975108898568763,
+      "grad_norm": 2.920839548110962,
+      "learning_rate": 1.8911439114391145e-06,
+      "loss": 1.3693,
+      "step": 1605
+    },
+    {
+      "epoch": 1.9987554449284382,
+      "grad_norm": 2.9545562267303467,
+      "learning_rate": 1.888837638376384e-06,
+      "loss": 1.4062,
+      "step": 1606
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 2.912815570831299,
+      "learning_rate": 1.8865313653136534e-06,
+      "loss": 1.4146,
+      "step": 1607
+    },
+    {
+      "epoch": 2.001244555071562,
+      "grad_norm": 3.1125903129577637,
+      "learning_rate": 1.8842250922509226e-06,
+      "loss": 1.3294,
+      "step": 1608
+    },
+    {
+      "epoch": 2.0024891101431237,
+      "grad_norm": 3.123905658721924,
+      "learning_rate": 1.881918819188192e-06,
+      "loss": 1.4175,
+      "step": 1609
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "grad_norm": 3.055100440979004,
+      "learning_rate": 1.8796125461254614e-06,
+      "loss": 1.3628,
+      "step": 1610
+    },
+    {
+      "epoch": 2.0037336652146855,
+      "eval_loss": 1.6198724508285522,
+      "eval_runtime": 42.5155,
+      "eval_samples_per_second": 23.521,
+      "eval_steps_per_second": 0.988,
+      "step": 1610
+    },
+    {
+      "epoch": 2.004978220286248,
+      "grad_norm": 3.087313413619995,
+      "learning_rate": 1.8773062730627308e-06,
+      "loss": 1.292,
+      "step": 1611
+    },
+    {
+      "epoch": 2.0062227753578097,
+      "grad_norm": 3.1849849224090576,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 1.3393,
+      "step": 1612
+    },
+    {
+      "epoch": 2.0074673304293715,
+      "grad_norm": 3.243750810623169,
+      "learning_rate": 1.8726937269372695e-06,
+      "loss": 1.3367,
+      "step": 1613
+    },
+    {
+      "epoch": 2.0087118855009334,
+      "grad_norm": 3.3086085319519043,
+      "learning_rate": 1.8703874538745389e-06,
+      "loss": 1.3099,
+      "step": 1614
+    },
+    {
+      "epoch": 2.0099564405724952,
+      "grad_norm": 3.388633966445923,
+      "learning_rate": 1.8680811808118083e-06,
+      "loss": 1.3816,
+      "step": 1615
+    },
+    {
+      "epoch": 2.011200995644057,
+      "grad_norm": 3.422671318054199,
+      "learning_rate": 1.8657749077490777e-06,
+      "loss": 1.3799,
+      "step": 1616
+    },
+    {
+      "epoch": 2.0124455507156194,
+      "grad_norm": 3.1104466915130615,
+      "learning_rate": 1.863468634686347e-06,
+      "loss": 1.3752,
+      "step": 1617
+    },
+    {
+      "epoch": 2.013690105787181,
+      "grad_norm": 3.2728488445281982,
+      "learning_rate": 1.8611623616236164e-06,
+      "loss": 1.322,
+      "step": 1618
+    },
+    {
+      "epoch": 2.014934660858743,
+      "grad_norm": 3.3357644081115723,
+      "learning_rate": 1.8588560885608858e-06,
+      "loss": 1.3193,
+      "step": 1619
+    },
+    {
+      "epoch": 2.016179215930305,
+      "grad_norm": 3.1026456356048584,
+      "learning_rate": 1.856549815498155e-06,
+      "loss": 1.3394,
+      "step": 1620
+    },
+    {
+      "epoch": 2.016179215930305,
+      "eval_loss": 1.6217585802078247,
+      "eval_runtime": 45.9794,
+      "eval_samples_per_second": 21.749,
+      "eval_steps_per_second": 0.913,
+      "step": 1620
+    },
+    {
+      "epoch": 2.0174237710018668,
+      "grad_norm": 3.314291000366211,
+      "learning_rate": 1.8542435424354244e-06,
+      "loss": 1.3778,
+      "step": 1621
+    },
+    {
+      "epoch": 2.0186683260734286,
+      "grad_norm": 3.3624770641326904,
+      "learning_rate": 1.8519372693726939e-06,
+      "loss": 1.3948,
+      "step": 1622
+    },
+    {
+      "epoch": 2.0199128811449905,
+      "grad_norm": 3.4041574001312256,
+      "learning_rate": 1.8496309963099633e-06,
+      "loss": 1.3109,
+      "step": 1623
+    },
+    {
+      "epoch": 2.0211574362165527,
+      "grad_norm": 3.5128726959228516,
+      "learning_rate": 1.8473247232472327e-06,
+      "loss": 1.4223,
+      "step": 1624
+    },
+    {
+      "epoch": 2.0224019912881146,
+      "grad_norm": 3.444768190383911,
+      "learning_rate": 1.845018450184502e-06,
+      "loss": 1.3672,
+      "step": 1625
+    },
+    {
+      "epoch": 2.0236465463596764,
+      "grad_norm": 3.478060722351074,
+      "learning_rate": 1.8427121771217713e-06,
+      "loss": 1.4384,
+      "step": 1626
+    },
+    {
+      "epoch": 2.0248911014312383,
+      "grad_norm": 3.211465835571289,
+      "learning_rate": 1.8404059040590408e-06,
+      "loss": 1.3891,
+      "step": 1627
+    },
+    {
+      "epoch": 2.0261356565028,
+      "grad_norm": 3.1283977031707764,
+      "learning_rate": 1.83809963099631e-06,
+      "loss": 1.3752,
+      "step": 1628
+    },
+    {
+      "epoch": 2.027380211574362,
+      "grad_norm": 3.3699934482574463,
+      "learning_rate": 1.8357933579335796e-06,
+      "loss": 1.3751,
+      "step": 1629
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "grad_norm": 3.0729823112487793,
+      "learning_rate": 1.8334870848708488e-06,
+      "loss": 1.3452,
+      "step": 1630
+    },
+    {
+      "epoch": 2.0286247666459243,
+      "eval_loss": 1.6185853481292725,
+      "eval_runtime": 45.2873,
+      "eval_samples_per_second": 22.081,
+      "eval_steps_per_second": 0.927,
+      "step": 1630
+    },
+    {
+      "epoch": 2.029869321717486,
+      "grad_norm": 3.274139642715454,
+      "learning_rate": 1.8311808118081182e-06,
+      "loss": 1.3242,
+      "step": 1631
+    },
+    {
+      "epoch": 2.031113876789048,
+      "grad_norm": 3.2869648933410645,
+      "learning_rate": 1.8288745387453877e-06,
+      "loss": 1.3435,
+      "step": 1632
+    },
+    {
+      "epoch": 2.03235843186061,
+      "grad_norm": 3.3614306449890137,
+      "learning_rate": 1.8265682656826569e-06,
+      "loss": 1.3705,
+      "step": 1633
+    },
+    {
+      "epoch": 2.0336029869321717,
+      "grad_norm": 3.411914587020874,
+      "learning_rate": 1.8242619926199265e-06,
+      "loss": 1.3942,
+      "step": 1634
+    },
+    {
+      "epoch": 2.0348475420037335,
+      "grad_norm": 3.274519920349121,
+      "learning_rate": 1.8219557195571957e-06,
+      "loss": 1.3393,
+      "step": 1635
+    },
+    {
+      "epoch": 2.0360920970752954,
+      "grad_norm": 3.993478775024414,
+      "learning_rate": 1.819649446494465e-06,
+      "loss": 1.3713,
+      "step": 1636
+    },
+    {
+      "epoch": 2.0373366521468577,
+      "grad_norm": 3.273571491241455,
+      "learning_rate": 1.8173431734317346e-06,
+      "loss": 1.3337,
+      "step": 1637
+    },
+    {
+      "epoch": 2.0385812072184195,
+      "grad_norm": 3.1542446613311768,
+      "learning_rate": 1.8150369003690038e-06,
+      "loss": 1.3793,
+      "step": 1638
+    },
+    {
+      "epoch": 2.0398257622899814,
+      "grad_norm": 3.263773202896118,
+      "learning_rate": 1.812730627306273e-06,
+      "loss": 1.3844,
+      "step": 1639
+    },
+    {
+      "epoch": 2.041070317361543,
+      "grad_norm": 3.268772840499878,
+      "learning_rate": 1.8104243542435426e-06,
+      "loss": 1.3947,
+      "step": 1640
+    },
+    {
+      "epoch": 2.041070317361543,
+      "eval_loss": 1.6149734258651733,
+      "eval_runtime": 48.5557,
+      "eval_samples_per_second": 20.595,
+      "eval_steps_per_second": 0.865,
+      "step": 1640
+    },
+    {
+      "epoch": 2.042314872433105,
+      "grad_norm": 3.190168619155884,
+      "learning_rate": 1.8081180811808118e-06,
+      "loss": 1.3276,
+      "step": 1641
+    },
+    {
+      "epoch": 2.043559427504667,
+      "grad_norm": 3.1573212146759033,
+      "learning_rate": 1.8058118081180815e-06,
+      "loss": 1.3307,
+      "step": 1642
+    },
+    {
+      "epoch": 2.044803982576229,
+      "grad_norm": 3.286513328552246,
+      "learning_rate": 1.8035055350553507e-06,
+      "loss": 1.3359,
+      "step": 1643
+    },
+    {
+      "epoch": 2.046048537647791,
+      "grad_norm": 3.342024803161621,
+      "learning_rate": 1.8011992619926199e-06,
+      "loss": 1.3472,
+      "step": 1644
+    },
+    {
+      "epoch": 2.047293092719353,
+      "grad_norm": 3.3237056732177734,
+      "learning_rate": 1.7988929889298895e-06,
+      "loss": 1.3898,
+      "step": 1645
+    },
+    {
+      "epoch": 2.0485376477909147,
+      "grad_norm": 3.370035409927368,
+      "learning_rate": 1.7965867158671587e-06,
+      "loss": 1.3537,
+      "step": 1646
+    },
+    {
+      "epoch": 2.0497822028624766,
+      "grad_norm": 3.1636598110198975,
+      "learning_rate": 1.7942804428044284e-06,
+      "loss": 1.3455,
+      "step": 1647
+    },
+    {
+      "epoch": 2.0510267579340384,
+      "grad_norm": 3.239396810531616,
+      "learning_rate": 1.7919741697416976e-06,
+      "loss": 1.3212,
+      "step": 1648
+    },
+    {
+      "epoch": 2.0522713130056003,
+      "grad_norm": 3.2765328884124756,
+      "learning_rate": 1.7896678966789668e-06,
+      "loss": 1.396,
+      "step": 1649
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "grad_norm": 3.2259459495544434,
+      "learning_rate": 1.7873616236162364e-06,
+      "loss": 1.3614,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0535158680771626,
+      "eval_loss": 1.614806056022644,
+      "eval_runtime": 49.8704,
+      "eval_samples_per_second": 20.052,
+      "eval_steps_per_second": 0.842,
+      "step": 1650
+    },
+    {
+      "epoch": 2.0547604231487244,
+      "grad_norm": 3.257683277130127,
+      "learning_rate": 1.7850553505535056e-06,
+      "loss": 1.3663,
+      "step": 1651
+    },
+    {
+      "epoch": 2.0560049782202863,
+      "grad_norm": 3.292070150375366,
+      "learning_rate": 1.7827490774907748e-06,
+      "loss": 1.3548,
+      "step": 1652
+    },
+    {
+      "epoch": 2.057249533291848,
+      "grad_norm": 3.3517003059387207,
+      "learning_rate": 1.7804428044280445e-06,
+      "loss": 1.4015,
+      "step": 1653
+    },
+    {
+      "epoch": 2.05849408836341,
+      "grad_norm": 3.5100908279418945,
+      "learning_rate": 1.7781365313653137e-06,
+      "loss": 1.3666,
+      "step": 1654
+    },
+    {
+      "epoch": 2.059738643434972,
+      "grad_norm": 3.3054633140563965,
+      "learning_rate": 1.7758302583025833e-06,
+      "loss": 1.3691,
+      "step": 1655
+    },
+    {
+      "epoch": 2.060983198506534,
+      "grad_norm": 3.1036577224731445,
+      "learning_rate": 1.7735239852398525e-06,
+      "loss": 1.2721,
+      "step": 1656
+    },
+    {
+      "epoch": 2.062227753578096,
+      "grad_norm": 3.604623317718506,
+      "learning_rate": 1.7712177121771217e-06,
+      "loss": 1.376,
+      "step": 1657
+    },
+    {
+      "epoch": 2.063472308649658,
+      "grad_norm": 3.5246458053588867,
+      "learning_rate": 1.7689114391143914e-06,
+      "loss": 1.3891,
+      "step": 1658
+    },
+    {
+      "epoch": 2.0647168637212197,
+      "grad_norm": 3.286940336227417,
+      "learning_rate": 1.7666051660516606e-06,
+      "loss": 1.3978,
+      "step": 1659
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "grad_norm": 3.1022047996520996,
+      "learning_rate": 1.7642988929889302e-06,
+      "loss": 1.3373,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0659614187927815,
+      "eval_loss": 1.6144907474517822,
+      "eval_runtime": 48.4382,
+      "eval_samples_per_second": 20.645,
+      "eval_steps_per_second": 0.867,
+      "step": 1660
+    },
+    {
+      "epoch": 2.0672059738643433,
+      "grad_norm": 3.4111568927764893,
+      "learning_rate": 1.7619926199261994e-06,
+      "loss": 1.3816,
+      "step": 1661
+    },
+    {
+      "epoch": 2.068450528935905,
+      "grad_norm": 3.478947162628174,
+      "learning_rate": 1.7596863468634687e-06,
+      "loss": 1.3601,
+      "step": 1662
+    },
+    {
+      "epoch": 2.0696950840074675,
+      "grad_norm": 3.3320810794830322,
+      "learning_rate": 1.7573800738007383e-06,
+      "loss": 1.3454,
+      "step": 1663
+    },
+    {
+      "epoch": 2.0709396390790293,
+      "grad_norm": 3.475529909133911,
+      "learning_rate": 1.7550738007380075e-06,
+      "loss": 1.3666,
+      "step": 1664
+    },
+    {
+      "epoch": 2.072184194150591,
+      "grad_norm": 3.302342653274536,
+      "learning_rate": 1.752767527675277e-06,
+      "loss": 1.3824,
+      "step": 1665
+    },
+    {
+      "epoch": 2.073428749222153,
+      "grad_norm": 3.198641777038574,
+      "learning_rate": 1.7504612546125463e-06,
+      "loss": 1.3398,
+      "step": 1666
+    },
+    {
+      "epoch": 2.074673304293715,
+      "grad_norm": 3.421834945678711,
+      "learning_rate": 1.7481549815498156e-06,
+      "loss": 1.363,
+      "step": 1667
+    },
+    {
+      "epoch": 2.0759178593652767,
+      "grad_norm": 3.654343843460083,
+      "learning_rate": 1.7458487084870852e-06,
+      "loss": 1.3852,
+      "step": 1668
+    },
+    {
+      "epoch": 2.077162414436839,
+      "grad_norm": 3.3543035984039307,
+      "learning_rate": 1.7435424354243544e-06,
+      "loss": 1.3554,
+      "step": 1669
+    },
+    {
+      "epoch": 2.078406969508401,
+      "grad_norm": 3.2144811153411865,
+      "learning_rate": 1.7412361623616236e-06,
+      "loss": 1.3137,
+      "step": 1670
+    },
+    {
+      "epoch": 2.078406969508401,
+      "eval_loss": 1.6155465841293335,
+      "eval_runtime": 52.4111,
+      "eval_samples_per_second": 19.08,
+      "eval_steps_per_second": 0.801,
+      "step": 1670
+    },
+    {
+      "epoch": 2.0796515245799627,
+      "grad_norm": 3.2240991592407227,
+      "learning_rate": 1.7389298892988932e-06,
+      "loss": 1.3578,
+      "step": 1671
+    },
+    {
+      "epoch": 2.0808960796515246,
+      "grad_norm": 3.1450018882751465,
+      "learning_rate": 1.7366236162361625e-06,
+      "loss": 1.2999,
+      "step": 1672
+    },
+    {
+      "epoch": 2.0821406347230864,
+      "grad_norm": 3.4396843910217285,
+      "learning_rate": 1.7343173431734319e-06,
+      "loss": 1.3522,
+      "step": 1673
+    },
+    {
+      "epoch": 2.0833851897946483,
+      "grad_norm": 3.1719677448272705,
+      "learning_rate": 1.7320110701107013e-06,
+      "loss": 1.3552,
+      "step": 1674
+    },
+    {
+      "epoch": 2.08462974486621,
+      "grad_norm": 3.4420199394226074,
+      "learning_rate": 1.7297047970479705e-06,
+      "loss": 1.4033,
+      "step": 1675
+    },
+    {
+      "epoch": 2.0858742999377724,
+      "grad_norm": 3.3366570472717285,
+      "learning_rate": 1.7273985239852401e-06,
+      "loss": 1.3407,
+      "step": 1676
+    },
+    {
+      "epoch": 2.0871188550093343,
+      "grad_norm": 3.3416335582733154,
+      "learning_rate": 1.7250922509225094e-06,
+      "loss": 1.3882,
+      "step": 1677
+    },
+    {
+      "epoch": 2.088363410080896,
+      "grad_norm": 3.4189534187316895,
+      "learning_rate": 1.7227859778597788e-06,
+      "loss": 1.3996,
+      "step": 1678
+    },
+    {
+      "epoch": 2.089607965152458,
+      "grad_norm": 3.309095859527588,
+      "learning_rate": 1.7204797047970482e-06,
+      "loss": 1.3799,
+      "step": 1679
+    },
+    {
+      "epoch": 2.09085252022402,
+      "grad_norm": 3.369511365890503,
+      "learning_rate": 1.7181734317343174e-06,
+      "loss": 1.4159,
+      "step": 1680
+    },
+    {
+      "epoch": 2.09085252022402,
+      "eval_loss": 1.6145448684692383,
+      "eval_runtime": 49.0228,
+      "eval_samples_per_second": 20.399,
+      "eval_steps_per_second": 0.857,
+      "step": 1680
+    },
+    {
+      "epoch": 2.0920970752955816,
+      "grad_norm": 3.323286771774292,
+      "learning_rate": 1.7158671586715868e-06,
+      "loss": 1.4035,
+      "step": 1681
+    },
+    {
+      "epoch": 2.093341630367144,
+      "grad_norm": 3.3053157329559326,
+      "learning_rate": 1.7135608856088563e-06,
+      "loss": 1.3498,
+      "step": 1682
+    },
+    {
+      "epoch": 2.094586185438706,
+      "grad_norm": 3.141890048980713,
+      "learning_rate": 1.7112546125461257e-06,
+      "loss": 1.3438,
+      "step": 1683
+    },
+    {
+      "epoch": 2.0958307405102676,
+      "grad_norm": 3.135058879852295,
+      "learning_rate": 1.708948339483395e-06,
+      "loss": 1.3726,
+      "step": 1684
+    },
+    {
+      "epoch": 2.0970752955818295,
+      "grad_norm": 3.2389063835144043,
+      "learning_rate": 1.7066420664206643e-06,
+      "loss": 1.3661,
+      "step": 1685
+    },
+    {
+      "epoch": 2.0983198506533913,
+      "grad_norm": 3.1741721630096436,
+      "learning_rate": 1.7043357933579337e-06,
+      "loss": 1.3438,
+      "step": 1686
+    },
+    {
+      "epoch": 2.099564405724953,
+      "grad_norm": 3.3524587154388428,
+      "learning_rate": 1.7020295202952032e-06,
+      "loss": 1.3388,
+      "step": 1687
+    },
+    {
+      "epoch": 2.100808960796515,
+      "grad_norm": 3.3574740886688232,
+      "learning_rate": 1.6997232472324724e-06,
+      "loss": 1.3702,
+      "step": 1688
+    },
+    {
+      "epoch": 2.1020535158680773,
+      "grad_norm": 3.458645820617676,
+      "learning_rate": 1.6974169741697418e-06,
+      "loss": 1.3785,
+      "step": 1689
+    },
+    {
+      "epoch": 2.103298070939639,
+      "grad_norm": 3.346677303314209,
+      "learning_rate": 1.6951107011070112e-06,
+      "loss": 1.3295,
+      "step": 1690
+    },
+    {
+      "epoch": 2.103298070939639,
+      "eval_loss": 1.6184555292129517,
+      "eval_runtime": 46.6544,
+      "eval_samples_per_second": 21.434,
+      "eval_steps_per_second": 0.9,
+      "step": 1690
+    },
+    {
+      "epoch": 2.104542626011201,
+      "grad_norm": 3.3403804302215576,
+      "learning_rate": 1.6928044280442806e-06,
+      "loss": 1.3912,
+      "step": 1691
+    },
+    {
+      "epoch": 2.105787181082763,
+      "grad_norm": 3.2638344764709473,
+      "learning_rate": 1.6904981549815499e-06,
+      "loss": 1.3684,
+      "step": 1692
+    },
+    {
+      "epoch": 2.1070317361543247,
+      "grad_norm": 3.192701578140259,
+      "learning_rate": 1.6881918819188193e-06,
+      "loss": 1.2997,
+      "step": 1693
+    },
+    {
+      "epoch": 2.1082762912258866,
+      "grad_norm": 3.482121706008911,
+      "learning_rate": 1.6858856088560887e-06,
+      "loss": 1.3515,
+      "step": 1694
+    },
+    {
+      "epoch": 2.109520846297449,
+      "grad_norm": 3.292353630065918,
+      "learning_rate": 1.6835793357933581e-06,
+      "loss": 1.3603,
+      "step": 1695
+    },
+    {
+      "epoch": 2.1107654013690107,
+      "grad_norm": 3.34220290184021,
+      "learning_rate": 1.6812730627306275e-06,
+      "loss": 1.3666,
+      "step": 1696
+    },
+    {
+      "epoch": 2.1120099564405725,
+      "grad_norm": 3.190058469772339,
+      "learning_rate": 1.6789667896678968e-06,
+      "loss": 1.3549,
+      "step": 1697
+    },
+    {
+      "epoch": 2.1132545115121344,
+      "grad_norm": 3.3759775161743164,
+      "learning_rate": 1.6766605166051662e-06,
+      "loss": 1.3628,
+      "step": 1698
+    },
+    {
+      "epoch": 2.1144990665836962,
+      "grad_norm": 3.419440507888794,
+      "learning_rate": 1.6743542435424356e-06,
+      "loss": 1.3618,
+      "step": 1699
+    },
+    {
+      "epoch": 2.115743621655258,
+      "grad_norm": 3.5021324157714844,
+      "learning_rate": 1.6720479704797048e-06,
+      "loss": 1.4023,
+      "step": 1700
+    },
+    {
+      "epoch": 2.115743621655258,
+      "eval_loss": 1.6218053102493286,
+      "eval_runtime": 48.3483,
+      "eval_samples_per_second": 20.683,
+      "eval_steps_per_second": 0.869,
+      "step": 1700
+    },
+    {
+      "epoch": 2.11698817672682,
+      "grad_norm": 3.449636936187744,
+      "learning_rate": 1.6697416974169742e-06,
+      "loss": 1.3534,
+      "step": 1701
+    },
+    {
+      "epoch": 2.1182327317983822,
+      "grad_norm": 3.483173370361328,
+      "learning_rate": 1.6674354243542437e-06,
+      "loss": 1.4053,
+      "step": 1702
+    },
+    {
+      "epoch": 2.119477286869944,
+      "grad_norm": 3.3713502883911133,
+      "learning_rate": 1.665129151291513e-06,
+      "loss": 1.3352,
+      "step": 1703
+    },
+    {
+      "epoch": 2.120721841941506,
+      "grad_norm": 3.2565910816192627,
+      "learning_rate": 1.6628228782287825e-06,
+      "loss": 1.3751,
+      "step": 1704
+    },
+    {
+      "epoch": 2.1219663970130678,
+      "grad_norm": 3.1434268951416016,
+      "learning_rate": 1.6605166051660517e-06,
+      "loss": 1.346,
+      "step": 1705
+    },
+    {
+      "epoch": 2.1232109520846296,
+      "grad_norm": 3.229909896850586,
+      "learning_rate": 1.6582103321033211e-06,
+      "loss": 1.3477,
+      "step": 1706
+    },
+    {
+      "epoch": 2.1244555071561915,
+      "grad_norm": 3.3930044174194336,
+      "learning_rate": 1.6559040590405906e-06,
+      "loss": 1.3374,
+      "step": 1707
+    },
+    {
+      "epoch": 2.1257000622277538,
+      "grad_norm": 3.2674202919006348,
+      "learning_rate": 1.6535977859778598e-06,
+      "loss": 1.321,
+      "step": 1708
+    },
+    {
+      "epoch": 2.1269446172993156,
+      "grad_norm": 3.3048641681671143,
+      "learning_rate": 1.6512915129151294e-06,
+      "loss": 1.3641,
+      "step": 1709
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "grad_norm": 3.4714198112487793,
+      "learning_rate": 1.6489852398523986e-06,
+      "loss": 1.3931,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1281891723708775,
+      "eval_loss": 1.6198205947875977,
+      "eval_runtime": 42.9272,
+      "eval_samples_per_second": 23.295,
+      "eval_steps_per_second": 0.978,
+      "step": 1710
+    },
+    {
+      "epoch": 2.1294337274424393,
+      "grad_norm": 3.440058946609497,
+      "learning_rate": 1.6466789667896678e-06,
+      "loss": 1.3892,
+      "step": 1711
+    },
+    {
+      "epoch": 2.130678282514001,
+      "grad_norm": 3.4067635536193848,
+      "learning_rate": 1.6443726937269375e-06,
+      "loss": 1.3719,
+      "step": 1712
+    },
+    {
+      "epoch": 2.131922837585563,
+      "grad_norm": 3.285536766052246,
+      "learning_rate": 1.6420664206642067e-06,
+      "loss": 1.3735,
+      "step": 1713
+    },
+    {
+      "epoch": 2.133167392657125,
+      "grad_norm": 3.171344041824341,
+      "learning_rate": 1.6397601476014763e-06,
+      "loss": 1.3628,
+      "step": 1714
+    },
+    {
+      "epoch": 2.134411947728687,
+      "grad_norm": 3.3051321506500244,
+      "learning_rate": 1.6374538745387455e-06,
+      "loss": 1.3579,
+      "step": 1715
+    },
+    {
+      "epoch": 2.135656502800249,
+      "grad_norm": 3.2083475589752197,
+      "learning_rate": 1.6351476014760147e-06,
+      "loss": 1.297,
+      "step": 1716
+    },
+    {
+      "epoch": 2.136901057871811,
+      "grad_norm": 3.340172052383423,
+      "learning_rate": 1.6328413284132844e-06,
+      "loss": 1.3468,
+      "step": 1717
+    },
+    {
+      "epoch": 2.1381456129433727,
+      "grad_norm": 3.3475770950317383,
+      "learning_rate": 1.6305350553505536e-06,
+      "loss": 1.3918,
+      "step": 1718
+    },
+    {
+      "epoch": 2.1393901680149345,
+      "grad_norm": 3.4087703227996826,
+      "learning_rate": 1.6282287822878228e-06,
+      "loss": 1.4083,
+      "step": 1719
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "grad_norm": 3.461160898208618,
+      "learning_rate": 1.6259225092250924e-06,
+      "loss": 1.3471,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1406347230864964,
+      "eval_loss": 1.6216135025024414,
+      "eval_runtime": 49.1097,
+      "eval_samples_per_second": 20.363,
+      "eval_steps_per_second": 0.855,
+      "step": 1720
+    },
+    {
+      "epoch": 2.1418792781580587,
+      "grad_norm": 3.3619799613952637,
+      "learning_rate": 1.6236162361623616e-06,
+      "loss": 1.3235,
+      "step": 1721
+    },
+    {
+      "epoch": 2.1431238332296205,
+      "grad_norm": 3.332010507583618,
+      "learning_rate": 1.6213099630996313e-06,
+      "loss": 1.3576,
+      "step": 1722
+    },
+    {
+      "epoch": 2.1443683883011824,
+      "grad_norm": 3.300055503845215,
+      "learning_rate": 1.6190036900369005e-06,
+      "loss": 1.3269,
+      "step": 1723
+    },
+    {
+      "epoch": 2.145612943372744,
+      "grad_norm": 3.3326926231384277,
+      "learning_rate": 1.6166974169741697e-06,
+      "loss": 1.346,
+      "step": 1724
+    },
+    {
+      "epoch": 2.146857498444306,
+      "grad_norm": 3.4713706970214844,
+      "learning_rate": 1.6143911439114393e-06,
+      "loss": 1.3416,
+      "step": 1725
+    },
+    {
+      "epoch": 2.148102053515868,
+      "grad_norm": 3.1899967193603516,
+      "learning_rate": 1.6120848708487085e-06,
+      "loss": 1.3192,
+      "step": 1726
+    },
+    {
+      "epoch": 2.1493466085874298,
+      "grad_norm": 3.1781857013702393,
+      "learning_rate": 1.6097785977859782e-06,
+      "loss": 1.3197,
+      "step": 1727
+    },
+    {
+      "epoch": 2.150591163658992,
+      "grad_norm": 3.3369524478912354,
+      "learning_rate": 1.6074723247232474e-06,
+      "loss": 1.3804,
+      "step": 1728
+    },
+    {
+      "epoch": 2.151835718730554,
+      "grad_norm": 3.2813405990600586,
+      "learning_rate": 1.6051660516605166e-06,
+      "loss": 1.3778,
+      "step": 1729
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "grad_norm": 3.3395135402679443,
+      "learning_rate": 1.6028597785977862e-06,
+      "loss": 1.3523,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1530802738021158,
+      "eval_loss": 1.6181366443634033,
+      "eval_runtime": 50.7133,
+      "eval_samples_per_second": 19.719,
+      "eval_steps_per_second": 0.828,
+      "step": 1730
+    },
+    {
+      "epoch": 2.1543248288736776,
+      "grad_norm": 3.2905092239379883,
+      "learning_rate": 1.6005535055350554e-06,
+      "loss": 1.3568,
+      "step": 1731
+    },
+    {
+      "epoch": 2.1555693839452394,
+      "grad_norm": 3.314086437225342,
+      "learning_rate": 1.5982472324723247e-06,
+      "loss": 1.3543,
+      "step": 1732
+    },
+    {
+      "epoch": 2.1568139390168013,
+      "grad_norm": 3.340034008026123,
+      "learning_rate": 1.5959409594095943e-06,
+      "loss": 1.3639,
+      "step": 1733
+    },
+    {
+      "epoch": 2.1580584940883636,
+      "grad_norm": 3.264814615249634,
+      "learning_rate": 1.5936346863468635e-06,
+      "loss": 1.3599,
+      "step": 1734
+    },
+    {
+      "epoch": 2.1593030491599254,
+      "grad_norm": 3.2429778575897217,
+      "learning_rate": 1.5913284132841331e-06,
+      "loss": 1.3379,
+      "step": 1735
+    },
+    {
+      "epoch": 2.1605476042314873,
+      "grad_norm": 3.221935987472534,
+      "learning_rate": 1.5890221402214023e-06,
+      "loss": 1.3162,
+      "step": 1736
+    },
+    {
+      "epoch": 2.161792159303049,
+      "grad_norm": 3.266073703765869,
+      "learning_rate": 1.5867158671586716e-06,
+      "loss": 1.3632,
+      "step": 1737
+    },
+    {
+      "epoch": 2.163036714374611,
+      "grad_norm": 3.2618112564086914,
+      "learning_rate": 1.5844095940959412e-06,
+      "loss": 1.3779,
+      "step": 1738
+    },
+    {
+      "epoch": 2.164281269446173,
+      "grad_norm": 3.2474825382232666,
+      "learning_rate": 1.5821033210332104e-06,
+      "loss": 1.3201,
+      "step": 1739
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "grad_norm": 3.4494247436523438,
+      "learning_rate": 1.57979704797048e-06,
+      "loss": 1.3949,
+      "step": 1740
+    },
+    {
+      "epoch": 2.1655258245177347,
+      "eval_loss": 1.6180604696273804,
+      "eval_runtime": 49.3152,
+      "eval_samples_per_second": 20.278,
+      "eval_steps_per_second": 0.852,
+      "step": 1740
+    },
+    {
+      "epoch": 2.166770379589297,
+      "grad_norm": 3.221304416656494,
+      "learning_rate": 1.5774907749077492e-06,
+      "loss": 1.3341,
+      "step": 1741
+    },
+    {
+      "epoch": 2.168014934660859,
+      "grad_norm": 3.28930401802063,
+      "learning_rate": 1.5751845018450185e-06,
+      "loss": 1.3151,
+      "step": 1742
+    },
+    {
+      "epoch": 2.1692594897324207,
+      "grad_norm": 3.3582026958465576,
+      "learning_rate": 1.572878228782288e-06,
+      "loss": 1.3767,
+      "step": 1743
+    },
+    {
+      "epoch": 2.1705040448039825,
+      "grad_norm": 3.2603683471679688,
+      "learning_rate": 1.5705719557195573e-06,
+      "loss": 1.3644,
+      "step": 1744
+    },
+    {
+      "epoch": 2.1717485998755444,
+      "grad_norm": 3.3291070461273193,
+      "learning_rate": 1.5682656826568267e-06,
+      "loss": 1.3226,
+      "step": 1745
+    },
+    {
+      "epoch": 2.172993154947106,
+      "grad_norm": 3.473127841949463,
+      "learning_rate": 1.5659594095940962e-06,
+      "loss": 1.3531,
+      "step": 1746
+    },
+    {
+      "epoch": 2.1742377100186685,
+      "grad_norm": 3.2014408111572266,
+      "learning_rate": 1.5636531365313654e-06,
+      "loss": 1.3634,
+      "step": 1747
+    },
+    {
+      "epoch": 2.1754822650902303,
+      "grad_norm": 3.56447434425354,
+      "learning_rate": 1.561346863468635e-06,
+      "loss": 1.3788,
+      "step": 1748
+    },
+    {
+      "epoch": 2.176726820161792,
+      "grad_norm": 3.351318359375,
+      "learning_rate": 1.5590405904059042e-06,
+      "loss": 1.3133,
+      "step": 1749
+    },
+    {
+      "epoch": 2.177971375233354,
+      "grad_norm": 3.278183937072754,
+      "learning_rate": 1.5567343173431734e-06,
+      "loss": 1.3164,
+      "step": 1750
+    },
+    {
+      "epoch": 2.177971375233354,
+      "eval_loss": 1.6197904348373413,
+      "eval_runtime": 50.2776,
+      "eval_samples_per_second": 19.89,
+      "eval_steps_per_second": 0.835,
+      "step": 1750
+    },
+    {
+      "epoch": 2.179215930304916,
+      "grad_norm": 3.379854917526245,
+      "learning_rate": 1.554428044280443e-06,
+      "loss": 1.3785,
+      "step": 1751
+    },
+    {
+      "epoch": 2.1804604853764777,
+      "grad_norm": 3.2265610694885254,
+      "learning_rate": 1.5521217712177123e-06,
+      "loss": 1.3483,
+      "step": 1752
+    },
+    {
+      "epoch": 2.18170504044804,
+      "grad_norm": 3.2259609699249268,
+      "learning_rate": 1.5498154981549817e-06,
+      "loss": 1.377,
+      "step": 1753
+    },
+    {
+      "epoch": 2.182949595519602,
+      "grad_norm": 3.1671273708343506,
+      "learning_rate": 1.5475092250922511e-06,
+      "loss": 1.3106,
+      "step": 1754
+    },
+    {
+      "epoch": 2.1841941505911637,
+      "grad_norm": 3.280299425125122,
+      "learning_rate": 1.5452029520295203e-06,
+      "loss": 1.3271,
+      "step": 1755
+    },
+    {
+      "epoch": 2.1854387056627256,
+      "grad_norm": 3.224238634109497,
+      "learning_rate": 1.5428966789667897e-06,
+      "loss": 1.3369,
+      "step": 1756
+    },
+    {
+      "epoch": 2.1866832607342874,
+      "grad_norm": 3.3139021396636963,
+      "learning_rate": 1.5405904059040592e-06,
+      "loss": 1.3562,
+      "step": 1757
+    },
+    {
+      "epoch": 2.1879278158058493,
+      "grad_norm": 3.340569257736206,
+      "learning_rate": 1.5382841328413286e-06,
+      "loss": 1.3526,
+      "step": 1758
+    },
+    {
+      "epoch": 2.189172370877411,
+      "grad_norm": 3.505159854888916,
+      "learning_rate": 1.535977859778598e-06,
+      "loss": 1.3796,
+      "step": 1759
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "grad_norm": 3.351260185241699,
+      "learning_rate": 1.5336715867158672e-06,
+      "loss": 1.3676,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1904169259489734,
+      "eval_loss": 1.6169729232788086,
+      "eval_runtime": 48.0801,
+      "eval_samples_per_second": 20.799,
+      "eval_steps_per_second": 0.874,
+      "step": 1760
+    },
+    {
+      "epoch": 2.1916614810205353,
+      "grad_norm": 3.4291462898254395,
+      "learning_rate": 1.5313653136531366e-06,
+      "loss": 1.3574,
+      "step": 1761
+    },
+    {
+      "epoch": 2.192906036092097,
+      "grad_norm": 3.5459227561950684,
+      "learning_rate": 1.529059040590406e-06,
+      "loss": 1.3919,
+      "step": 1762
+    },
+    {
+      "epoch": 2.194150591163659,
+      "grad_norm": 3.298430919647217,
+      "learning_rate": 1.5267527675276755e-06,
+      "loss": 1.3551,
+      "step": 1763
+    },
+    {
+      "epoch": 2.195395146235221,
+      "grad_norm": 3.4521286487579346,
+      "learning_rate": 1.5244464944649447e-06,
+      "loss": 1.3779,
+      "step": 1764
+    },
+    {
+      "epoch": 2.1966397013067827,
+      "grad_norm": 3.354443073272705,
+      "learning_rate": 1.5221402214022141e-06,
+      "loss": 1.3574,
+      "step": 1765
+    },
+    {
+      "epoch": 2.197884256378345,
+      "grad_norm": 3.069854259490967,
+      "learning_rate": 1.5198339483394836e-06,
+      "loss": 1.2987,
+      "step": 1766
+    },
+    {
+      "epoch": 2.199128811449907,
+      "grad_norm": 3.0894274711608887,
+      "learning_rate": 1.517527675276753e-06,
+      "loss": 1.3408,
+      "step": 1767
+    },
+    {
+      "epoch": 2.2003733665214686,
+      "grad_norm": 3.3033676147460938,
+      "learning_rate": 1.5152214022140222e-06,
+      "loss": 1.3394,
+      "step": 1768
+    },
+    {
+      "epoch": 2.2016179215930305,
+      "grad_norm": 3.2894108295440674,
+      "learning_rate": 1.5129151291512916e-06,
+      "loss": 1.3505,
+      "step": 1769
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "grad_norm": 3.263564109802246,
+      "learning_rate": 1.510608856088561e-06,
+      "loss": 1.3318,
+      "step": 1770
+    },
+    {
+      "epoch": 2.2028624766645923,
+      "eval_loss": 1.6183674335479736,
+      "eval_runtime": 48.3189,
+      "eval_samples_per_second": 20.696,
+      "eval_steps_per_second": 0.869,
+      "step": 1770
+    },
+    {
+      "epoch": 2.204107031736154,
+      "grad_norm": 3.2933449745178223,
+      "learning_rate": 1.5083025830258305e-06,
+      "loss": 1.3397,
+      "step": 1771
+    },
+    {
+      "epoch": 2.205351586807716,
+      "grad_norm": 3.3185815811157227,
+      "learning_rate": 1.5059963099630997e-06,
+      "loss": 1.3339,
+      "step": 1772
+    },
+    {
+      "epoch": 2.2065961418792783,
+      "grad_norm": 3.445251703262329,
+      "learning_rate": 1.503690036900369e-06,
+      "loss": 1.3898,
+      "step": 1773
+    },
+    {
+      "epoch": 2.20784069695084,
+      "grad_norm": 3.399627685546875,
+      "learning_rate": 1.5013837638376385e-06,
+      "loss": 1.3646,
+      "step": 1774
+    },
+    {
+      "epoch": 2.209085252022402,
+      "grad_norm": 3.3483574390411377,
+      "learning_rate": 1.499077490774908e-06,
+      "loss": 1.3732,
+      "step": 1775
+    },
+    {
+      "epoch": 2.210329807093964,
+      "grad_norm": 3.3097689151763916,
+      "learning_rate": 1.4967712177121774e-06,
+      "loss": 1.3317,
+      "step": 1776
+    },
+    {
+      "epoch": 2.2115743621655257,
+      "grad_norm": 3.254025459289551,
+      "learning_rate": 1.4944649446494466e-06,
+      "loss": 1.3818,
+      "step": 1777
+    },
+    {
+      "epoch": 2.2128189172370876,
+      "grad_norm": 3.2063064575195312,
+      "learning_rate": 1.492158671586716e-06,
+      "loss": 1.3357,
+      "step": 1778
+    },
+    {
+      "epoch": 2.21406347230865,
+      "grad_norm": 3.1974005699157715,
+      "learning_rate": 1.4898523985239854e-06,
+      "loss": 1.4103,
+      "step": 1779
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "grad_norm": 3.1551153659820557,
+      "learning_rate": 1.4875461254612546e-06,
+      "loss": 1.3088,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2153080273802117,
+      "eval_loss": 1.6195781230926514,
+      "eval_runtime": 52.379,
+      "eval_samples_per_second": 19.092,
+      "eval_steps_per_second": 0.802,
+      "step": 1780
+    },
+    {
+      "epoch": 2.2165525824517736,
+      "grad_norm": 3.3186404705047607,
+      "learning_rate": 1.485239852398524e-06,
+      "loss": 1.3775,
+      "step": 1781
+    },
+    {
+      "epoch": 2.2177971375233354,
+      "grad_norm": 3.2764358520507812,
+      "learning_rate": 1.4829335793357935e-06,
+      "loss": 1.3253,
+      "step": 1782
+    },
+    {
+      "epoch": 2.2190416925948973,
+      "grad_norm": 3.3535802364349365,
+      "learning_rate": 1.4806273062730627e-06,
+      "loss": 1.3451,
+      "step": 1783
+    },
+    {
+      "epoch": 2.220286247666459,
+      "grad_norm": 3.304255962371826,
+      "learning_rate": 1.4783210332103323e-06,
+      "loss": 1.3565,
+      "step": 1784
+    },
+    {
+      "epoch": 2.2215308027380214,
+      "grad_norm": 3.281244993209839,
+      "learning_rate": 1.4760147601476015e-06,
+      "loss": 1.3458,
+      "step": 1785
+    },
+    {
+      "epoch": 2.2227753578095832,
+      "grad_norm": 3.4199814796447754,
+      "learning_rate": 1.473708487084871e-06,
+      "loss": 1.359,
+      "step": 1786
+    },
+    {
+      "epoch": 2.224019912881145,
+      "grad_norm": 3.298577070236206,
+      "learning_rate": 1.4714022140221404e-06,
+      "loss": 1.3298,
+      "step": 1787
+    },
+    {
+      "epoch": 2.225264467952707,
+      "grad_norm": 3.2206099033355713,
+      "learning_rate": 1.4690959409594096e-06,
+      "loss": 1.3026,
+      "step": 1788
+    },
+    {
+      "epoch": 2.226509023024269,
+      "grad_norm": 3.3583920001983643,
+      "learning_rate": 1.4667896678966792e-06,
+      "loss": 1.3618,
+      "step": 1789
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "grad_norm": 3.4446728229522705,
+      "learning_rate": 1.4644833948339484e-06,
+      "loss": 1.3895,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2277535780958306,
+      "eval_loss": 1.6163197755813599,
+      "eval_runtime": 49.4171,
+      "eval_samples_per_second": 20.236,
+      "eval_steps_per_second": 0.85,
+      "step": 1790
+    },
+    {
+      "epoch": 2.2289981331673925,
+      "grad_norm": 3.329155445098877,
+      "learning_rate": 1.4621771217712176e-06,
+      "loss": 1.3368,
+      "step": 1791
+    },
+    {
+      "epoch": 2.2302426882389548,
+      "grad_norm": 3.4599087238311768,
+      "learning_rate": 1.4598708487084873e-06,
+      "loss": 1.3487,
+      "step": 1792
+    },
+    {
+      "epoch": 2.2314872433105166,
+      "grad_norm": 3.3162965774536133,
+      "learning_rate": 1.4575645756457565e-06,
+      "loss": 1.3493,
+      "step": 1793
+    },
+    {
+      "epoch": 2.2327317983820785,
+      "grad_norm": 3.1206021308898926,
+      "learning_rate": 1.4552583025830261e-06,
+      "loss": 1.3506,
+      "step": 1794
+    },
+    {
+      "epoch": 2.2339763534536403,
+      "grad_norm": 3.334165573120117,
+      "learning_rate": 1.4529520295202953e-06,
+      "loss": 1.3315,
+      "step": 1795
+    },
+    {
+      "epoch": 2.235220908525202,
+      "grad_norm": 3.3404204845428467,
+      "learning_rate": 1.4506457564575645e-06,
+      "loss": 1.3163,
+      "step": 1796
+    },
+    {
+      "epoch": 2.236465463596764,
+      "grad_norm": 3.2753002643585205,
+      "learning_rate": 1.4483394833948342e-06,
+      "loss": 1.3567,
+      "step": 1797
+    },
+    {
+      "epoch": 2.2377100186683263,
+      "grad_norm": 3.42242431640625,
+      "learning_rate": 1.4460332103321034e-06,
+      "loss": 1.3761,
+      "step": 1798
+    },
+    {
+      "epoch": 2.238954573739888,
+      "grad_norm": 3.31380033493042,
+      "learning_rate": 1.4437269372693726e-06,
+      "loss": 1.3353,
+      "step": 1799
+    },
+    {
+      "epoch": 2.24019912881145,
+      "grad_norm": 3.239811658859253,
+      "learning_rate": 1.4414206642066422e-06,
+      "loss": 1.3187,
+      "step": 1800
+    },
+    {
+      "epoch": 2.24019912881145,
+      "eval_loss": 1.6201411485671997,
+      "eval_runtime": 53.5369,
+      "eval_samples_per_second": 18.679,
+      "eval_steps_per_second": 0.785,
+      "step": 1800
+    },
+    {
+      "epoch": 2.241443683883012,
+      "grad_norm": 3.549010753631592,
+      "learning_rate": 1.4391143911439114e-06,
+      "loss": 1.3633,
+      "step": 1801
+    },
+    {
+      "epoch": 2.2426882389545737,
+      "grad_norm": 3.1888253688812256,
+      "learning_rate": 1.436808118081181e-06,
+      "loss": 1.3292,
+      "step": 1802
+    },
+    {
+      "epoch": 2.2439327940261355,
+      "grad_norm": 3.1015641689300537,
+      "learning_rate": 1.4345018450184503e-06,
+      "loss": 1.3404,
+      "step": 1803
+    },
+    {
+      "epoch": 2.2451773490976974,
+      "grad_norm": 3.2635490894317627,
+      "learning_rate": 1.4321955719557195e-06,
+      "loss": 1.3181,
+      "step": 1804
+    },
+    {
+      "epoch": 2.2464219041692597,
+      "grad_norm": 3.314697265625,
+      "learning_rate": 1.4298892988929891e-06,
+      "loss": 1.308,
+      "step": 1805
+    },
+    {
+      "epoch": 2.2476664592408215,
+      "grad_norm": 3.3572986125946045,
+      "learning_rate": 1.4275830258302584e-06,
+      "loss": 1.2888,
+      "step": 1806
+    },
+    {
+      "epoch": 2.2489110143123834,
+      "grad_norm": 3.403423547744751,
+      "learning_rate": 1.425276752767528e-06,
+      "loss": 1.3438,
+      "step": 1807
+    },
+    {
+      "epoch": 2.2501555693839452,
+      "grad_norm": 3.5180821418762207,
+      "learning_rate": 1.4229704797047972e-06,
+      "loss": 1.3613,
+      "step": 1808
+    },
+    {
+      "epoch": 2.251400124455507,
+      "grad_norm": 3.5315089225769043,
+      "learning_rate": 1.4206642066420664e-06,
+      "loss": 1.3655,
+      "step": 1809
+    },
+    {
+      "epoch": 2.252644679527069,
+      "grad_norm": 3.4266903400421143,
+      "learning_rate": 1.418357933579336e-06,
+      "loss": 1.3481,
+      "step": 1810
+    },
+    {
+      "epoch": 2.252644679527069,
+      "eval_loss": 1.619051456451416,
+      "eval_runtime": 43.1595,
+      "eval_samples_per_second": 23.17,
+      "eval_steps_per_second": 0.973,
+      "step": 1810
+    },
+    {
+      "epoch": 2.253889234598631,
+      "grad_norm": 3.412053108215332,
+      "learning_rate": 1.4160516605166053e-06,
+      "loss": 1.3097,
+      "step": 1811
+    },
+    {
+      "epoch": 2.255133789670193,
+      "grad_norm": 3.4371933937072754,
+      "learning_rate": 1.4137453874538745e-06,
+      "loss": 1.3831,
+      "step": 1812
+    },
+    {
+      "epoch": 2.256378344741755,
+      "grad_norm": 3.331648588180542,
+      "learning_rate": 1.411439114391144e-06,
+      "loss": 1.3701,
+      "step": 1813
+    },
+    {
+      "epoch": 2.2576228998133168,
+      "grad_norm": 3.3840672969818115,
+      "learning_rate": 1.4091328413284133e-06,
+      "loss": 1.3676,
+      "step": 1814
+    },
+    {
+      "epoch": 2.2588674548848786,
+      "grad_norm": 3.3289988040924072,
+      "learning_rate": 1.406826568265683e-06,
+      "loss": 1.3281,
+      "step": 1815
+    },
+    {
+      "epoch": 2.2601120099564405,
+      "grad_norm": 3.309511423110962,
+      "learning_rate": 1.4045202952029522e-06,
+      "loss": 1.3216,
+      "step": 1816
+    },
+    {
+      "epoch": 2.2613565650280023,
+      "grad_norm": 3.3434019088745117,
+      "learning_rate": 1.4022140221402214e-06,
+      "loss": 1.3261,
+      "step": 1817
+    },
+    {
+      "epoch": 2.2626011200995646,
+      "grad_norm": 3.3371620178222656,
+      "learning_rate": 1.399907749077491e-06,
+      "loss": 1.3597,
+      "step": 1818
+    },
+    {
+      "epoch": 2.2638456751711264,
+      "grad_norm": 3.287353515625,
+      "learning_rate": 1.3976014760147602e-06,
+      "loss": 1.3431,
+      "step": 1819
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "grad_norm": 3.422102689743042,
+      "learning_rate": 1.3952952029520298e-06,
+      "loss": 1.3454,
+      "step": 1820
+    },
+    {
+      "epoch": 2.2650902302426883,
+      "eval_loss": 1.6200023889541626,
+      "eval_runtime": 46.3983,
+      "eval_samples_per_second": 21.552,
+      "eval_steps_per_second": 0.905,
+      "step": 1820
+    },
+    {
+      "epoch": 2.26633478531425,
+      "grad_norm": 3.6252498626708984,
+      "learning_rate": 1.392988929889299e-06,
+      "loss": 1.4066,
+      "step": 1821
+    },
+    {
+      "epoch": 2.267579340385812,
+      "grad_norm": 3.423708915710449,
+      "learning_rate": 1.3906826568265683e-06,
+      "loss": 1.3504,
+      "step": 1822
+    },
+    {
+      "epoch": 2.268823895457374,
+      "grad_norm": 3.3063361644744873,
+      "learning_rate": 1.388376383763838e-06,
+      "loss": 1.3635,
+      "step": 1823
+    },
+    {
+      "epoch": 2.270068450528936,
+      "grad_norm": 3.380657911300659,
+      "learning_rate": 1.3860701107011071e-06,
+      "loss": 1.3442,
+      "step": 1824
+    },
+    {
+      "epoch": 2.271313005600498,
+      "grad_norm": 3.300102472305298,
+      "learning_rate": 1.3837638376383765e-06,
+      "loss": 1.3741,
+      "step": 1825
+    },
+    {
+      "epoch": 2.27255756067206,
+      "grad_norm": 3.3895652294158936,
+      "learning_rate": 1.381457564575646e-06,
+      "loss": 1.3288,
+      "step": 1826
+    },
+    {
+      "epoch": 2.2738021157436217,
+      "grad_norm": 3.4393489360809326,
+      "learning_rate": 1.3791512915129152e-06,
+      "loss": 1.3544,
+      "step": 1827
+    },
+    {
+      "epoch": 2.2750466708151835,
+      "grad_norm": 3.4510536193847656,
+      "learning_rate": 1.3768450184501846e-06,
+      "loss": 1.3894,
+      "step": 1828
+    },
+    {
+      "epoch": 2.2762912258867454,
+      "grad_norm": 3.39282488822937,
+      "learning_rate": 1.374538745387454e-06,
+      "loss": 1.3776,
+      "step": 1829
+    },
+    {
+      "epoch": 2.277535780958307,
+      "grad_norm": 3.250432014465332,
+      "learning_rate": 1.3722324723247232e-06,
+      "loss": 1.3511,
+      "step": 1830
+    },
+    {
+      "epoch": 2.277535780958307,
+      "eval_loss": 1.6178572177886963,
+      "eval_runtime": 45.3086,
+      "eval_samples_per_second": 22.071,
+      "eval_steps_per_second": 0.927,
+      "step": 1830
+    },
+    {
+      "epoch": 2.2787803360298695,
+      "grad_norm": 3.4044647216796875,
+      "learning_rate": 1.3699261992619929e-06,
+      "loss": 1.3548,
+      "step": 1831
+    },
+    {
+      "epoch": 2.2800248911014314,
+      "grad_norm": 3.4996840953826904,
+      "learning_rate": 1.367619926199262e-06,
+      "loss": 1.4069,
+      "step": 1832
+    },
+    {
+      "epoch": 2.281269446172993,
+      "grad_norm": 3.4036638736724854,
+      "learning_rate": 1.3653136531365315e-06,
+      "loss": 1.3499,
+      "step": 1833
+    },
+    {
+      "epoch": 2.282514001244555,
+      "grad_norm": 3.328805685043335,
+      "learning_rate": 1.363007380073801e-06,
+      "loss": 1.3266,
+      "step": 1834
+    },
+    {
+      "epoch": 2.283758556316117,
+      "grad_norm": 3.410924196243286,
+      "learning_rate": 1.3607011070110701e-06,
+      "loss": 1.3303,
+      "step": 1835
+    },
+    {
+      "epoch": 2.2850031113876788,
+      "grad_norm": 3.323162794113159,
+      "learning_rate": 1.3583948339483396e-06,
+      "loss": 1.3284,
+      "step": 1836
+    },
+    {
+      "epoch": 2.286247666459241,
+      "grad_norm": 3.3356032371520996,
+      "learning_rate": 1.356088560885609e-06,
+      "loss": 1.2842,
+      "step": 1837
+    },
+    {
+      "epoch": 2.287492221530803,
+      "grad_norm": 3.3366599082946777,
+      "learning_rate": 1.3537822878228784e-06,
+      "loss": 1.321,
+      "step": 1838
+    },
+    {
+      "epoch": 2.2887367766023647,
+      "grad_norm": 3.3824799060821533,
+      "learning_rate": 1.3514760147601478e-06,
+      "loss": 1.3744,
+      "step": 1839
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "grad_norm": 3.6268486976623535,
+      "learning_rate": 1.349169741697417e-06,
+      "loss": 1.4097,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2899813316739266,
+      "eval_loss": 1.6148018836975098,
+      "eval_runtime": 48.3459,
+      "eval_samples_per_second": 20.684,
+      "eval_steps_per_second": 0.869,
+      "step": 1840
+    },
+    {
+      "epoch": 2.2912258867454884,
+      "grad_norm": 3.3530380725860596,
+      "learning_rate": 1.3468634686346865e-06,
+      "loss": 1.3924,
+      "step": 1841
+    },
+    {
+      "epoch": 2.2924704418170503,
+      "grad_norm": 3.456149101257324,
+      "learning_rate": 1.3445571955719559e-06,
+      "loss": 1.3837,
+      "step": 1842
+    },
+    {
+      "epoch": 2.293714996888612,
+      "grad_norm": 3.37180495262146,
+      "learning_rate": 1.3422509225092253e-06,
+      "loss": 1.3411,
+      "step": 1843
+    },
+    {
+      "epoch": 2.2949595519601744,
+      "grad_norm": 3.286172389984131,
+      "learning_rate": 1.3399446494464945e-06,
+      "loss": 1.2888,
+      "step": 1844
+    },
+    {
+      "epoch": 2.2962041070317363,
+      "grad_norm": 3.3966121673583984,
+      "learning_rate": 1.337638376383764e-06,
+      "loss": 1.3644,
+      "step": 1845
+    },
+    {
+      "epoch": 2.297448662103298,
+      "grad_norm": 3.4066531658172607,
+      "learning_rate": 1.3353321033210334e-06,
+      "loss": 1.3543,
+      "step": 1846
+    },
+    {
+      "epoch": 2.29869321717486,
+      "grad_norm": 3.542680501937866,
+      "learning_rate": 1.3330258302583028e-06,
+      "loss": 1.3823,
+      "step": 1847
+    },
+    {
+      "epoch": 2.299937772246422,
+      "grad_norm": 3.323636770248413,
+      "learning_rate": 1.330719557195572e-06,
+      "loss": 1.3031,
+      "step": 1848
+    },
+    {
+      "epoch": 2.3011823273179837,
+      "grad_norm": 3.4802908897399902,
+      "learning_rate": 1.3284132841328414e-06,
+      "loss": 1.4001,
+      "step": 1849
+    },
+    {
+      "epoch": 2.302426882389546,
+      "grad_norm": 3.310987710952759,
+      "learning_rate": 1.3261070110701108e-06,
+      "loss": 1.3385,
+      "step": 1850
+    },
+    {
+      "epoch": 2.302426882389546,
+      "eval_loss": 1.615319848060608,
+      "eval_runtime": 44.8343,
+      "eval_samples_per_second": 22.304,
+      "eval_steps_per_second": 0.937,
+      "step": 1850
+    },
+    {
+      "epoch": 2.303671437461108,
+      "grad_norm": 3.3461661338806152,
+      "learning_rate": 1.3238007380073803e-06,
+      "loss": 1.3461,
+      "step": 1851
+    },
+    {
+      "epoch": 2.3049159925326697,
+      "grad_norm": 3.317770481109619,
+      "learning_rate": 1.3214944649446495e-06,
+      "loss": 1.338,
+      "step": 1852
+    },
+    {
+      "epoch": 2.3061605476042315,
+      "grad_norm": 3.3806679248809814,
+      "learning_rate": 1.319188191881919e-06,
+      "loss": 1.3367,
+      "step": 1853
+    },
+    {
+      "epoch": 2.3074051026757934,
+      "grad_norm": 3.366933822631836,
+      "learning_rate": 1.3168819188191883e-06,
+      "loss": 1.3717,
+      "step": 1854
+    },
+    {
+      "epoch": 2.308649657747355,
+      "grad_norm": 3.3250250816345215,
+      "learning_rate": 1.3145756457564575e-06,
+      "loss": 1.3585,
+      "step": 1855
+    },
+    {
+      "epoch": 2.309894212818917,
+      "grad_norm": 3.320171356201172,
+      "learning_rate": 1.3122693726937272e-06,
+      "loss": 1.3491,
+      "step": 1856
+    },
+    {
+      "epoch": 2.3111387678904793,
+      "grad_norm": 3.418029546737671,
+      "learning_rate": 1.3099630996309964e-06,
+      "loss": 1.3627,
+      "step": 1857
+    },
+    {
+      "epoch": 2.312383322962041,
+      "grad_norm": 3.3762004375457764,
+      "learning_rate": 1.3076568265682658e-06,
+      "loss": 1.3409,
+      "step": 1858
+    },
+    {
+      "epoch": 2.313627878033603,
+      "grad_norm": 3.3099405765533447,
+      "learning_rate": 1.3053505535055352e-06,
+      "loss": 1.3593,
+      "step": 1859
+    },
+    {
+      "epoch": 2.314872433105165,
+      "grad_norm": 3.4060535430908203,
+      "learning_rate": 1.3030442804428044e-06,
+      "loss": 1.3363,
+      "step": 1860
+    },
+    {
+      "epoch": 2.314872433105165,
+      "eval_loss": 1.6176154613494873,
+      "eval_runtime": 46.8022,
+      "eval_samples_per_second": 21.367,
+      "eval_steps_per_second": 0.897,
+      "step": 1860
+    },
+    {
+      "epoch": 2.3161169881767267,
+      "grad_norm": 3.2827699184417725,
+      "learning_rate": 1.3007380073800739e-06,
+      "loss": 1.2718,
+      "step": 1861
+    },
+    {
+      "epoch": 2.3173615432482886,
+      "grad_norm": 3.5225489139556885,
+      "learning_rate": 1.2984317343173433e-06,
+      "loss": 1.3728,
+      "step": 1862
+    },
+    {
+      "epoch": 2.318606098319851,
+      "grad_norm": 3.475665807723999,
+      "learning_rate": 1.2961254612546125e-06,
+      "loss": 1.3772,
+      "step": 1863
+    },
+    {
+      "epoch": 2.3198506533914127,
+      "grad_norm": 3.385239839553833,
+      "learning_rate": 1.2938191881918821e-06,
+      "loss": 1.3471,
+      "step": 1864
+    },
+    {
+      "epoch": 2.3210952084629746,
+      "grad_norm": 3.312469005584717,
+      "learning_rate": 1.2915129151291513e-06,
+      "loss": 1.2961,
+      "step": 1865
+    },
+    {
+      "epoch": 2.3223397635345364,
+      "grad_norm": 3.3883018493652344,
+      "learning_rate": 1.2892066420664208e-06,
+      "loss": 1.3778,
+      "step": 1866
+    },
+    {
+      "epoch": 2.3235843186060983,
+      "grad_norm": 3.368964195251465,
+      "learning_rate": 1.2869003690036902e-06,
+      "loss": 1.3153,
+      "step": 1867
+    },
+    {
+      "epoch": 2.32482887367766,
+      "grad_norm": 3.4132297039031982,
+      "learning_rate": 1.2845940959409594e-06,
+      "loss": 1.3718,
+      "step": 1868
+    },
+    {
+      "epoch": 2.326073428749222,
+      "grad_norm": 3.3750340938568115,
+      "learning_rate": 1.282287822878229e-06,
+      "loss": 1.4035,
+      "step": 1869
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "grad_norm": 3.372588872909546,
+      "learning_rate": 1.2799815498154982e-06,
+      "loss": 1.3591,
+      "step": 1870
+    },
+    {
+      "epoch": 2.3273179838207843,
+      "eval_loss": 1.6196452379226685,
+      "eval_runtime": 43.8125,
+      "eval_samples_per_second": 22.825,
+      "eval_steps_per_second": 0.959,
+      "step": 1870
+    },
+    {
+      "epoch": 2.328562538892346,
+      "grad_norm": 3.341944694519043,
+      "learning_rate": 1.2776752767527675e-06,
+      "loss": 1.3239,
+      "step": 1871
+    },
+    {
+      "epoch": 2.329807093963908,
+      "grad_norm": 3.4855759143829346,
+      "learning_rate": 1.275369003690037e-06,
+      "loss": 1.3712,
+      "step": 1872
+    },
+    {
+      "epoch": 2.33105164903547,
+      "grad_norm": 3.577047348022461,
+      "learning_rate": 1.2730627306273063e-06,
+      "loss": 1.3502,
+      "step": 1873
+    },
+    {
+      "epoch": 2.3322962041070316,
+      "grad_norm": 3.479374885559082,
+      "learning_rate": 1.270756457564576e-06,
+      "loss": 1.3296,
+      "step": 1874
+    },
+    {
+      "epoch": 2.3335407591785935,
+      "grad_norm": 3.2203609943389893,
+      "learning_rate": 1.2684501845018451e-06,
+      "loss": 1.3105,
+      "step": 1875
+    },
+    {
+      "epoch": 2.334785314250156,
+      "grad_norm": 3.3177664279937744,
+      "learning_rate": 1.2661439114391144e-06,
+      "loss": 1.3365,
+      "step": 1876
+    },
+    {
+      "epoch": 2.3360298693217176,
+      "grad_norm": 3.224146842956543,
+      "learning_rate": 1.263837638376384e-06,
+      "loss": 1.3172,
+      "step": 1877
+    },
+    {
+      "epoch": 2.3372744243932795,
+      "grad_norm": 3.336106538772583,
+      "learning_rate": 1.2615313653136532e-06,
+      "loss": 1.3248,
+      "step": 1878
+    },
+    {
+      "epoch": 2.3385189794648413,
+      "grad_norm": 3.4414947032928467,
+      "learning_rate": 1.2592250922509224e-06,
+      "loss": 1.3637,
+      "step": 1879
+    },
+    {
+      "epoch": 2.339763534536403,
+      "grad_norm": 3.5528085231781006,
+      "learning_rate": 1.256918819188192e-06,
+      "loss": 1.3986,
+      "step": 1880
+    },
+    {
+      "epoch": 2.339763534536403,
+      "eval_loss": 1.6137934923171997,
+      "eval_runtime": 43.8859,
+      "eval_samples_per_second": 22.786,
+      "eval_steps_per_second": 0.957,
+      "step": 1880
+    },
+    {
+      "epoch": 2.341008089607965,
+      "grad_norm": 3.406446695327759,
+      "learning_rate": 1.2546125461254613e-06,
+      "loss": 1.3479,
+      "step": 1881
+    },
+    {
+      "epoch": 2.342252644679527,
+      "grad_norm": 3.4129297733306885,
+      "learning_rate": 1.2523062730627309e-06,
+      "loss": 1.3227,
+      "step": 1882
+    },
+    {
+      "epoch": 2.343497199751089,
+      "grad_norm": 3.3013577461242676,
+      "learning_rate": 1.25e-06,
+      "loss": 1.3277,
+      "step": 1883
+    },
+    {
+      "epoch": 2.344741754822651,
+      "grad_norm": 3.3039028644561768,
+      "learning_rate": 1.2476937269372695e-06,
+      "loss": 1.3208,
+      "step": 1884
+    },
+    {
+      "epoch": 2.345986309894213,
+      "grad_norm": 3.3239917755126953,
+      "learning_rate": 1.245387453874539e-06,
+      "loss": 1.3293,
+      "step": 1885
+    },
+    {
+      "epoch": 2.3472308649657747,
+      "grad_norm": 3.239821195602417,
+      "learning_rate": 1.2430811808118082e-06,
+      "loss": 1.3191,
+      "step": 1886
+    },
+    {
+      "epoch": 2.3484754200373366,
+      "grad_norm": 3.6869564056396484,
+      "learning_rate": 1.2407749077490776e-06,
+      "loss": 1.363,
+      "step": 1887
+    },
+    {
+      "epoch": 2.3497199751088984,
+      "grad_norm": 3.5775156021118164,
+      "learning_rate": 1.238468634686347e-06,
+      "loss": 1.3564,
+      "step": 1888
+    },
+    {
+      "epoch": 2.3509645301804607,
+      "grad_norm": 3.3177807331085205,
+      "learning_rate": 1.2361623616236164e-06,
+      "loss": 1.3044,
+      "step": 1889
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "grad_norm": 3.622527837753296,
+      "learning_rate": 1.2338560885608856e-06,
+      "loss": 1.381,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3522090852520225,
+      "eval_loss": 1.6165976524353027,
+      "eval_runtime": 44.4026,
+      "eval_samples_per_second": 22.521,
+      "eval_steps_per_second": 0.946,
+      "step": 1890
+    },
+    {
+      "epoch": 2.3534536403235844,
+      "grad_norm": 3.3163886070251465,
+      "learning_rate": 1.231549815498155e-06,
+      "loss": 1.3598,
+      "step": 1891
+    },
+    {
+      "epoch": 2.3546981953951462,
+      "grad_norm": 3.3344099521636963,
+      "learning_rate": 1.2292435424354245e-06,
+      "loss": 1.3733,
+      "step": 1892
+    },
+    {
+      "epoch": 2.355942750466708,
+      "grad_norm": 3.259413480758667,
+      "learning_rate": 1.226937269372694e-06,
+      "loss": 1.3281,
+      "step": 1893
+    },
+    {
+      "epoch": 2.35718730553827,
+      "grad_norm": 3.427147150039673,
+      "learning_rate": 1.2246309963099633e-06,
+      "loss": 1.3484,
+      "step": 1894
+    },
+    {
+      "epoch": 2.358431860609832,
+      "grad_norm": 3.3890154361724854,
+      "learning_rate": 1.2223247232472325e-06,
+      "loss": 1.3768,
+      "step": 1895
+    },
+    {
+      "epoch": 2.359676415681394,
+      "grad_norm": 3.2845499515533447,
+      "learning_rate": 1.220018450184502e-06,
+      "loss": 1.3579,
+      "step": 1896
+    },
+    {
+      "epoch": 2.360920970752956,
+      "grad_norm": 3.3046178817749023,
+      "learning_rate": 1.2177121771217714e-06,
+      "loss": 1.3322,
+      "step": 1897
+    },
+    {
+      "epoch": 2.3621655258245178,
+      "grad_norm": 3.4520273208618164,
+      "learning_rate": 1.2154059040590408e-06,
+      "loss": 1.3515,
+      "step": 1898
+    },
+    {
+      "epoch": 2.3634100808960796,
+      "grad_norm": 3.3665220737457275,
+      "learning_rate": 1.21309963099631e-06,
+      "loss": 1.3236,
+      "step": 1899
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "grad_norm": 3.3551578521728516,
+      "learning_rate": 1.2107933579335794e-06,
+      "loss": 1.3568,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3646546359676415,
+      "eval_loss": 1.6159367561340332,
+      "eval_runtime": 44.1216,
+      "eval_samples_per_second": 22.665,
+      "eval_steps_per_second": 0.952,
+      "step": 1900
+    },
+    {
+      "epoch": 2.3658991910392033,
+      "grad_norm": 3.6204397678375244,
+      "learning_rate": 1.2084870848708489e-06,
+      "loss": 1.4072,
+      "step": 1901
+    },
+    {
+      "epoch": 2.3671437461107656,
+      "grad_norm": 3.3525922298431396,
+      "learning_rate": 1.2061808118081183e-06,
+      "loss": 1.328,
+      "step": 1902
+    },
+    {
+      "epoch": 2.3683883011823275,
+      "grad_norm": 3.3686022758483887,
+      "learning_rate": 1.2038745387453875e-06,
+      "loss": 1.3086,
+      "step": 1903
+    },
+    {
+      "epoch": 2.3696328562538893,
+      "grad_norm": 3.5756561756134033,
+      "learning_rate": 1.201568265682657e-06,
+      "loss": 1.3784,
+      "step": 1904
+    },
+    {
+      "epoch": 2.370877411325451,
+      "grad_norm": 3.4186534881591797,
+      "learning_rate": 1.1992619926199263e-06,
+      "loss": 1.3546,
+      "step": 1905
+    },
+    {
+      "epoch": 2.372121966397013,
+      "grad_norm": 3.367868661880493,
+      "learning_rate": 1.1969557195571958e-06,
+      "loss": 1.3527,
+      "step": 1906
+    },
+    {
+      "epoch": 2.373366521468575,
+      "grad_norm": 3.3395912647247314,
+      "learning_rate": 1.1946494464944652e-06,
+      "loss": 1.3366,
+      "step": 1907
+    },
+    {
+      "epoch": 2.3746110765401367,
+      "grad_norm": 3.2967982292175293,
+      "learning_rate": 1.1923431734317344e-06,
+      "loss": 1.3229,
+      "step": 1908
+    },
+    {
+      "epoch": 2.375855631611699,
+      "grad_norm": 3.2422475814819336,
+      "learning_rate": 1.1900369003690038e-06,
+      "loss": 1.3455,
+      "step": 1909
+    },
+    {
+      "epoch": 2.377100186683261,
+      "grad_norm": 3.3781206607818604,
+      "learning_rate": 1.1877306273062733e-06,
+      "loss": 1.3314,
+      "step": 1910
+    },
+    {
+      "epoch": 2.377100186683261,
+      "eval_loss": 1.6179407835006714,
+      "eval_runtime": 42.9089,
+      "eval_samples_per_second": 23.305,
+      "eval_steps_per_second": 0.979,
+      "step": 1910
+    },
+    {
+      "epoch": 2.3783447417548227,
+      "grad_norm": 3.438586711883545,
+      "learning_rate": 1.1854243542435427e-06,
+      "loss": 1.3654,
+      "step": 1911
+    },
+    {
+      "epoch": 2.3795892968263845,
+      "grad_norm": 3.503896474838257,
+      "learning_rate": 1.1831180811808119e-06,
+      "loss": 1.3716,
+      "step": 1912
+    },
+    {
+      "epoch": 2.3808338518979464,
+      "grad_norm": 3.411320686340332,
+      "learning_rate": 1.1808118081180813e-06,
+      "loss": 1.3315,
+      "step": 1913
+    },
+    {
+      "epoch": 2.3820784069695082,
+      "grad_norm": 3.1860697269439697,
+      "learning_rate": 1.1785055350553507e-06,
+      "loss": 1.3016,
+      "step": 1914
+    },
+    {
+      "epoch": 2.3833229620410705,
+      "grad_norm": 3.4494664669036865,
+      "learning_rate": 1.1761992619926202e-06,
+      "loss": 1.3629,
+      "step": 1915
+    },
+    {
+      "epoch": 2.3845675171126324,
+      "grad_norm": 3.303277015686035,
+      "learning_rate": 1.1738929889298894e-06,
+      "loss": 1.3207,
+      "step": 1916
+    },
+    {
+      "epoch": 2.385812072184194,
+      "grad_norm": 3.254411458969116,
+      "learning_rate": 1.1715867158671588e-06,
+      "loss": 1.3477,
+      "step": 1917
+    },
+    {
+      "epoch": 2.387056627255756,
+      "grad_norm": 3.258666753768921,
+      "learning_rate": 1.1692804428044282e-06,
+      "loss": 1.3452,
+      "step": 1918
+    },
+    {
+      "epoch": 2.388301182327318,
+      "grad_norm": 3.3814001083374023,
+      "learning_rate": 1.1669741697416974e-06,
+      "loss": 1.329,
+      "step": 1919
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "grad_norm": 3.367011547088623,
+      "learning_rate": 1.1646678966789668e-06,
+      "loss": 1.3326,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3895457373988798,
+      "eval_loss": 1.6158324480056763,
+      "eval_runtime": 45.8088,
+      "eval_samples_per_second": 21.83,
+      "eval_steps_per_second": 0.917,
+      "step": 1920
+    },
+    {
+      "epoch": 2.3907902924704416,
+      "grad_norm": 3.313123941421509,
+      "learning_rate": 1.1623616236162363e-06,
+      "loss": 1.307,
+      "step": 1921
+    },
+    {
+      "epoch": 2.392034847542004,
+      "grad_norm": 3.27423357963562,
+      "learning_rate": 1.1600553505535057e-06,
+      "loss": 1.3249,
+      "step": 1922
+    },
+    {
+      "epoch": 2.3932794026135658,
+      "grad_norm": 3.2963404655456543,
+      "learning_rate": 1.157749077490775e-06,
+      "loss": 1.3394,
+      "step": 1923
+    },
+    {
+      "epoch": 2.3945239576851276,
+      "grad_norm": 3.387673854827881,
+      "learning_rate": 1.1554428044280443e-06,
+      "loss": 1.3586,
+      "step": 1924
+    },
+    {
+      "epoch": 2.3957685127566895,
+      "grad_norm": 3.2588143348693848,
+      "learning_rate": 1.1531365313653137e-06,
+      "loss": 1.317,
+      "step": 1925
+    },
+    {
+      "epoch": 2.3970130678282513,
+      "grad_norm": 3.285653829574585,
+      "learning_rate": 1.1508302583025832e-06,
+      "loss": 1.2847,
+      "step": 1926
+    },
+    {
+      "epoch": 2.398257622899813,
+      "grad_norm": 3.4495935440063477,
+      "learning_rate": 1.1485239852398524e-06,
+      "loss": 1.3815,
+      "step": 1927
+    },
+    {
+      "epoch": 2.3995021779713754,
+      "grad_norm": 3.287186622619629,
+      "learning_rate": 1.1462177121771218e-06,
+      "loss": 1.3048,
+      "step": 1928
+    },
+    {
+      "epoch": 2.4007467330429373,
+      "grad_norm": 3.318042278289795,
+      "learning_rate": 1.1439114391143912e-06,
+      "loss": 1.313,
+      "step": 1929
+    },
+    {
+      "epoch": 2.401991288114499,
+      "grad_norm": 3.2705087661743164,
+      "learning_rate": 1.1416051660516606e-06,
+      "loss": 1.3207,
+      "step": 1930
+    },
+    {
+      "epoch": 2.401991288114499,
+      "eval_loss": 1.613088846206665,
+      "eval_runtime": 46.0901,
+      "eval_samples_per_second": 21.697,
+      "eval_steps_per_second": 0.911,
+      "step": 1930
+    },
+    {
+      "epoch": 2.403235843186061,
+      "grad_norm": 3.307523250579834,
+      "learning_rate": 1.1392988929889299e-06,
+      "loss": 1.3312,
+      "step": 1931
+    },
+    {
+      "epoch": 2.404480398257623,
+      "grad_norm": 3.3040900230407715,
+      "learning_rate": 1.1369926199261993e-06,
+      "loss": 1.3279,
+      "step": 1932
+    },
+    {
+      "epoch": 2.4057249533291847,
+      "grad_norm": 3.232649087905884,
+      "learning_rate": 1.1346863468634687e-06,
+      "loss": 1.314,
+      "step": 1933
+    },
+    {
+      "epoch": 2.4069695084007465,
+      "grad_norm": 3.3581860065460205,
+      "learning_rate": 1.1323800738007381e-06,
+      "loss": 1.3158,
+      "step": 1934
+    },
+    {
+      "epoch": 2.408214063472309,
+      "grad_norm": 3.378774404525757,
+      "learning_rate": 1.1300738007380073e-06,
+      "loss": 1.3252,
+      "step": 1935
+    },
+    {
+      "epoch": 2.4094586185438707,
+      "grad_norm": 3.4180240631103516,
+      "learning_rate": 1.1277675276752768e-06,
+      "loss": 1.3606,
+      "step": 1936
+    },
+    {
+      "epoch": 2.4107031736154325,
+      "grad_norm": 3.410921335220337,
+      "learning_rate": 1.1254612546125462e-06,
+      "loss": 1.3484,
+      "step": 1937
+    },
+    {
+      "epoch": 2.4119477286869944,
+      "grad_norm": 3.4201443195343018,
+      "learning_rate": 1.1231549815498156e-06,
+      "loss": 1.3659,
+      "step": 1938
+    },
+    {
+      "epoch": 2.413192283758556,
+      "grad_norm": 3.381019115447998,
+      "learning_rate": 1.1208487084870848e-06,
+      "loss": 1.3261,
+      "step": 1939
+    },
+    {
+      "epoch": 2.414436838830118,
+      "grad_norm": 3.516425371170044,
+      "learning_rate": 1.1185424354243542e-06,
+      "loss": 1.3618,
+      "step": 1940
+    },
+    {
+      "epoch": 2.414436838830118,
+      "eval_loss": 1.6142985820770264,
+      "eval_runtime": 45.8868,
+      "eval_samples_per_second": 21.793,
+      "eval_steps_per_second": 0.915,
+      "step": 1940
+    },
+    {
+      "epoch": 2.4156813939016804,
+      "grad_norm": 3.5319905281066895,
+      "learning_rate": 1.1162361623616237e-06,
+      "loss": 1.338,
+      "step": 1941
+    },
+    {
+      "epoch": 2.416925948973242,
+      "grad_norm": 3.4355592727661133,
+      "learning_rate": 1.113929889298893e-06,
+      "loss": 1.3389,
+      "step": 1942
+    },
+    {
+      "epoch": 2.418170504044804,
+      "grad_norm": 3.362187623977661,
+      "learning_rate": 1.1116236162361623e-06,
+      "loss": 1.3194,
+      "step": 1943
+    },
+    {
+      "epoch": 2.419415059116366,
+      "grad_norm": 3.4991087913513184,
+      "learning_rate": 1.1093173431734317e-06,
+      "loss": 1.3758,
+      "step": 1944
+    },
+    {
+      "epoch": 2.4206596141879277,
+      "grad_norm": 3.5951290130615234,
+      "learning_rate": 1.1070110701107011e-06,
+      "loss": 1.3548,
+      "step": 1945
+    },
+    {
+      "epoch": 2.4219041692594896,
+      "grad_norm": 3.3458032608032227,
+      "learning_rate": 1.1047047970479706e-06,
+      "loss": 1.3483,
+      "step": 1946
+    },
+    {
+      "epoch": 2.4231487243310514,
+      "grad_norm": 3.426600933074951,
+      "learning_rate": 1.10239852398524e-06,
+      "loss": 1.3548,
+      "step": 1947
+    },
+    {
+      "epoch": 2.4243932794026137,
+      "grad_norm": 3.5074410438537598,
+      "learning_rate": 1.1000922509225092e-06,
+      "loss": 1.3608,
+      "step": 1948
+    },
+    {
+      "epoch": 2.4256378344741756,
+      "grad_norm": 3.5622129440307617,
+      "learning_rate": 1.0977859778597786e-06,
+      "loss": 1.3861,
+      "step": 1949
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "grad_norm": 3.537771463394165,
+      "learning_rate": 1.095479704797048e-06,
+      "loss": 1.3408,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4268823895457374,
+      "eval_loss": 1.612324833869934,
+      "eval_runtime": 46.7327,
+      "eval_samples_per_second": 21.398,
+      "eval_steps_per_second": 0.899,
+      "step": 1950
+    },
+    {
+      "epoch": 2.4281269446172993,
+      "grad_norm": 3.430544137954712,
+      "learning_rate": 1.0931734317343175e-06,
+      "loss": 1.3678,
+      "step": 1951
+    },
+    {
+      "epoch": 2.429371499688861,
+      "grad_norm": 3.2243361473083496,
+      "learning_rate": 1.0908671586715867e-06,
+      "loss": 1.2982,
+      "step": 1952
+    },
+    {
+      "epoch": 2.430616054760423,
+      "grad_norm": 3.426236867904663,
+      "learning_rate": 1.088560885608856e-06,
+      "loss": 1.3827,
+      "step": 1953
+    },
+    {
+      "epoch": 2.4318606098319853,
+      "grad_norm": 3.3180603981018066,
+      "learning_rate": 1.0862546125461255e-06,
+      "loss": 1.3065,
+      "step": 1954
+    },
+    {
+      "epoch": 2.433105164903547,
+      "grad_norm": 3.3775084018707275,
+      "learning_rate": 1.083948339483395e-06,
+      "loss": 1.2759,
+      "step": 1955
+    },
+    {
+      "epoch": 2.434349719975109,
+      "grad_norm": 3.4124209880828857,
+      "learning_rate": 1.0816420664206644e-06,
+      "loss": 1.3571,
+      "step": 1956
+    },
+    {
+      "epoch": 2.435594275046671,
+      "grad_norm": 3.494408369064331,
+      "learning_rate": 1.0793357933579336e-06,
+      "loss": 1.3931,
+      "step": 1957
+    },
+    {
+      "epoch": 2.4368388301182327,
+      "grad_norm": 3.2699358463287354,
+      "learning_rate": 1.077029520295203e-06,
+      "loss": 1.3182,
+      "step": 1958
+    },
+    {
+      "epoch": 2.4380833851897945,
+      "grad_norm": 3.281592607498169,
+      "learning_rate": 1.0747232472324724e-06,
+      "loss": 1.2629,
+      "step": 1959
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "grad_norm": 3.3045406341552734,
+      "learning_rate": 1.0724169741697419e-06,
+      "loss": 1.3093,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4393279402613564,
+      "eval_loss": 1.6135376691818237,
+      "eval_runtime": 51.0587,
+      "eval_samples_per_second": 19.585,
+      "eval_steps_per_second": 0.823,
+      "step": 1960
+    },
+    {
+      "epoch": 2.4405724953329186,
+      "grad_norm": 3.3883392810821533,
+      "learning_rate": 1.070110701107011e-06,
+      "loss": 1.4005,
+      "step": 1961
+    },
+    {
+      "epoch": 2.4418170504044805,
+      "grad_norm": 3.3957419395446777,
+      "learning_rate": 1.0678044280442805e-06,
+      "loss": 1.3342,
+      "step": 1962
+    },
+    {
+      "epoch": 2.4430616054760423,
+      "grad_norm": 3.1982152462005615,
+      "learning_rate": 1.06549815498155e-06,
+      "loss": 1.3273,
+      "step": 1963
+    },
+    {
+      "epoch": 2.444306160547604,
+      "grad_norm": 3.3369827270507812,
+      "learning_rate": 1.0631918819188193e-06,
+      "loss": 1.312,
+      "step": 1964
+    },
+    {
+      "epoch": 2.445550715619166,
+      "grad_norm": 3.3651299476623535,
+      "learning_rate": 1.0608856088560888e-06,
+      "loss": 1.3728,
+      "step": 1965
+    },
+    {
+      "epoch": 2.446795270690728,
+      "grad_norm": 3.3393259048461914,
+      "learning_rate": 1.058579335793358e-06,
+      "loss": 1.3454,
+      "step": 1966
+    },
+    {
+      "epoch": 2.44803982576229,
+      "grad_norm": 3.4974067211151123,
+      "learning_rate": 1.0562730627306274e-06,
+      "loss": 1.3747,
+      "step": 1967
+    },
+    {
+      "epoch": 2.449284380833852,
+      "grad_norm": 3.4064528942108154,
+      "learning_rate": 1.0539667896678968e-06,
+      "loss": 1.3695,
+      "step": 1968
+    },
+    {
+      "epoch": 2.450528935905414,
+      "grad_norm": 3.347919225692749,
+      "learning_rate": 1.0516605166051662e-06,
+      "loss": 1.3439,
+      "step": 1969
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "grad_norm": 3.4505932331085205,
+      "learning_rate": 1.0493542435424354e-06,
+      "loss": 1.3723,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4517734909769757,
+      "eval_loss": 1.611203908920288,
+      "eval_runtime": 52.5131,
+      "eval_samples_per_second": 19.043,
+      "eval_steps_per_second": 0.8,
+      "step": 1970
+    },
+    {
+      "epoch": 2.4530180460485376,
+      "grad_norm": 3.4287805557250977,
+      "learning_rate": 1.0470479704797049e-06,
+      "loss": 1.3426,
+      "step": 1971
+    },
+    {
+      "epoch": 2.4542626011200994,
+      "grad_norm": 3.4337522983551025,
+      "learning_rate": 1.0447416974169743e-06,
+      "loss": 1.3451,
+      "step": 1972
+    },
+    {
+      "epoch": 2.4555071561916613,
+      "grad_norm": 3.4307212829589844,
+      "learning_rate": 1.0424354243542437e-06,
+      "loss": 1.3097,
+      "step": 1973
+    },
+    {
+      "epoch": 2.4567517112632236,
+      "grad_norm": 3.307767868041992,
+      "learning_rate": 1.0401291512915131e-06,
+      "loss": 1.2751,
+      "step": 1974
+    },
+    {
+      "epoch": 2.4579962663347854,
+      "grad_norm": 3.4809703826904297,
+      "learning_rate": 1.0378228782287824e-06,
+      "loss": 1.3522,
+      "step": 1975
+    },
+    {
+      "epoch": 2.4592408214063473,
+      "grad_norm": 3.235170602798462,
+      "learning_rate": 1.0355166051660518e-06,
+      "loss": 1.3003,
+      "step": 1976
+    },
+    {
+      "epoch": 2.460485376477909,
+      "grad_norm": 3.2969000339508057,
+      "learning_rate": 1.0332103321033212e-06,
+      "loss": 1.2993,
+      "step": 1977
+    },
+    {
+      "epoch": 2.461729931549471,
+      "grad_norm": 3.311535358428955,
+      "learning_rate": 1.0309040590405906e-06,
+      "loss": 1.3262,
+      "step": 1978
+    },
+    {
+      "epoch": 2.462974486621033,
+      "grad_norm": 3.2320244312286377,
+      "learning_rate": 1.0285977859778598e-06,
+      "loss": 1.3111,
+      "step": 1979
+    },
+    {
+      "epoch": 2.464219041692595,
+      "grad_norm": 3.2673771381378174,
+      "learning_rate": 1.0262915129151293e-06,
+      "loss": 1.3439,
+      "step": 1980
+    },
+    {
+      "epoch": 2.464219041692595,
+      "eval_loss": 1.6103569269180298,
+      "eval_runtime": 47.2563,
+      "eval_samples_per_second": 21.161,
+      "eval_steps_per_second": 0.889,
+      "step": 1980
+    },
+    {
+      "epoch": 2.465463596764157,
+      "grad_norm": 3.324113130569458,
+      "learning_rate": 1.0239852398523987e-06,
+      "loss": 1.3278,
+      "step": 1981
+    },
+    {
+      "epoch": 2.466708151835719,
+      "grad_norm": 3.3066444396972656,
+      "learning_rate": 1.021678966789668e-06,
+      "loss": 1.3438,
+      "step": 1982
+    },
+    {
+      "epoch": 2.4679527069072806,
+      "grad_norm": 3.463088035583496,
+      "learning_rate": 1.0193726937269373e-06,
+      "loss": 1.3293,
+      "step": 1983
+    },
+    {
+      "epoch": 2.4691972619788425,
+      "grad_norm": 3.406008005142212,
+      "learning_rate": 1.0170664206642067e-06,
+      "loss": 1.4026,
+      "step": 1984
+    },
+    {
+      "epoch": 2.4704418170504043,
+      "grad_norm": 3.408823251724243,
+      "learning_rate": 1.0147601476014762e-06,
+      "loss": 1.3226,
+      "step": 1985
+    },
+    {
+      "epoch": 2.471686372121966,
+      "grad_norm": 3.2991373538970947,
+      "learning_rate": 1.0124538745387456e-06,
+      "loss": 1.2682,
+      "step": 1986
+    },
+    {
+      "epoch": 2.4729309271935285,
+      "grad_norm": 3.3868234157562256,
+      "learning_rate": 1.010147601476015e-06,
+      "loss": 1.3286,
+      "step": 1987
+    },
+    {
+      "epoch": 2.4741754822650903,
+      "grad_norm": 3.3641555309295654,
+      "learning_rate": 1.0078413284132842e-06,
+      "loss": 1.3423,
+      "step": 1988
+    },
+    {
+      "epoch": 2.475420037336652,
+      "grad_norm": 3.3237619400024414,
+      "learning_rate": 1.0055350553505536e-06,
+      "loss": 1.2979,
+      "step": 1989
+    },
+    {
+      "epoch": 2.476664592408214,
+      "grad_norm": 3.6278553009033203,
+      "learning_rate": 1.003228782287823e-06,
+      "loss": 1.3543,
+      "step": 1990
+    },
+    {
+      "epoch": 2.476664592408214,
+      "eval_loss": 1.6103031635284424,
+      "eval_runtime": 46.0573,
+      "eval_samples_per_second": 21.712,
+      "eval_steps_per_second": 0.912,
+      "step": 1990
+    },
+    {
+      "epoch": 2.477909147479776,
+      "grad_norm": 3.2861995697021484,
+      "learning_rate": 1.0009225092250923e-06,
+      "loss": 1.3303,
+      "step": 1991
+    },
+    {
+      "epoch": 2.4791537025513377,
+      "grad_norm": 3.352261781692505,
+      "learning_rate": 9.986162361623617e-07,
+      "loss": 1.3527,
+      "step": 1992
+    },
+    {
+      "epoch": 2.4803982576229,
+      "grad_norm": 3.483726978302002,
+      "learning_rate": 9.963099630996311e-07,
+      "loss": 1.3762,
+      "step": 1993
+    },
+    {
+      "epoch": 2.481642812694462,
+      "grad_norm": 3.357154607772827,
+      "learning_rate": 9.940036900369005e-07,
+      "loss": 1.3601,
+      "step": 1994
+    },
+    {
+      "epoch": 2.4828873677660237,
+      "grad_norm": 3.4477577209472656,
+      "learning_rate": 9.916974169741698e-07,
+      "loss": 1.3636,
+      "step": 1995
+    },
+    {
+      "epoch": 2.4841319228375855,
+      "grad_norm": 3.377279043197632,
+      "learning_rate": 9.893911439114392e-07,
+      "loss": 1.3349,
+      "step": 1996
+    },
+    {
+      "epoch": 2.4853764779091474,
+      "grad_norm": 3.2705130577087402,
+      "learning_rate": 9.870848708487086e-07,
+      "loss": 1.3659,
+      "step": 1997
+    },
+    {
+      "epoch": 2.4866210329807092,
+      "grad_norm": 3.418225049972534,
+      "learning_rate": 9.84778597785978e-07,
+      "loss": 1.3534,
+      "step": 1998
+    },
+    {
+      "epoch": 2.487865588052271,
+      "grad_norm": 3.2288014888763428,
+      "learning_rate": 9.824723247232472e-07,
+      "loss": 1.3392,
+      "step": 1999
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "grad_norm": 3.3513882160186768,
+      "learning_rate": 9.801660516605167e-07,
+      "loss": 1.3483,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4891101431238334,
+      "eval_loss": 1.6108943223953247,
+      "eval_runtime": 48.275,
+      "eval_samples_per_second": 20.715,
+      "eval_steps_per_second": 0.87,
+      "step": 2000
+    },
+    {
+      "epoch": 2.4903546981953952,
+      "grad_norm": 3.35583758354187,
+      "learning_rate": 9.77859778597786e-07,
+      "loss": 1.3808,
+      "step": 2001
+    },
+    {
+      "epoch": 2.491599253266957,
+      "grad_norm": 3.4611411094665527,
+      "learning_rate": 9.755535055350555e-07,
+      "loss": 1.3512,
+      "step": 2002
+    },
+    {
+      "epoch": 2.492843808338519,
+      "grad_norm": 3.389965057373047,
+      "learning_rate": 9.732472324723247e-07,
+      "loss": 1.2777,
+      "step": 2003
+    },
+    {
+      "epoch": 2.4940883634100808,
+      "grad_norm": 3.370044231414795,
+      "learning_rate": 9.709409594095941e-07,
+      "loss": 1.3407,
+      "step": 2004
+    },
+    {
+      "epoch": 2.4953329184816426,
+      "grad_norm": 3.4178175926208496,
+      "learning_rate": 9.686346863468636e-07,
+      "loss": 1.3419,
+      "step": 2005
+    },
+    {
+      "epoch": 2.496577473553205,
+      "grad_norm": 3.4306576251983643,
+      "learning_rate": 9.66328413284133e-07,
+      "loss": 1.3029,
+      "step": 2006
+    },
+    {
+      "epoch": 2.4978220286247668,
+      "grad_norm": 3.4822888374328613,
+      "learning_rate": 9.640221402214022e-07,
+      "loss": 1.3512,
+      "step": 2007
+    },
+    {
+      "epoch": 2.4990665836963286,
+      "grad_norm": 3.2686867713928223,
+      "learning_rate": 9.617158671586716e-07,
+      "loss": 1.3366,
+      "step": 2008
+    },
+    {
+      "epoch": 2.5003111387678905,
+      "grad_norm": 3.413224220275879,
+      "learning_rate": 9.59409594095941e-07,
+      "loss": 1.324,
+      "step": 2009
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "grad_norm": 3.180180311203003,
+      "learning_rate": 9.571033210332105e-07,
+      "loss": 1.3054,
+      "step": 2010
+    },
+    {
+      "epoch": 2.5015556938394523,
+      "eval_loss": 1.6145573854446411,
+      "eval_runtime": 42.6598,
+      "eval_samples_per_second": 23.441,
+      "eval_steps_per_second": 0.985,
+      "step": 2010
+    },
+    {
+      "epoch": 2.502800248911014,
+      "grad_norm": 3.4968016147613525,
+      "learning_rate": 9.547970479704797e-07,
+      "loss": 1.3782,
+      "step": 2011
+    },
+    {
+      "epoch": 2.504044803982576,
+      "grad_norm": 3.270414113998413,
+      "learning_rate": 9.524907749077492e-07,
+      "loss": 1.3389,
+      "step": 2012
+    },
+    {
+      "epoch": 2.5052893590541383,
+      "grad_norm": 3.3850443363189697,
+      "learning_rate": 9.501845018450185e-07,
+      "loss": 1.3335,
+      "step": 2013
+    },
+    {
+      "epoch": 2.5065339141257,
+      "grad_norm": 3.3836278915405273,
+      "learning_rate": 9.478782287822879e-07,
+      "loss": 1.3343,
+      "step": 2014
+    },
+    {
+      "epoch": 2.507778469197262,
+      "grad_norm": 3.4950475692749023,
+      "learning_rate": 9.455719557195573e-07,
+      "loss": 1.337,
+      "step": 2015
+    },
+    {
+      "epoch": 2.509023024268824,
+      "grad_norm": 3.368262529373169,
+      "learning_rate": 9.432656826568267e-07,
+      "loss": 1.32,
+      "step": 2016
+    },
+    {
+      "epoch": 2.5102675793403857,
+      "grad_norm": 3.4199533462524414,
+      "learning_rate": 9.40959409594096e-07,
+      "loss": 1.3754,
+      "step": 2017
+    },
+    {
+      "epoch": 2.511512134411948,
+      "grad_norm": 3.3372180461883545,
+      "learning_rate": 9.386531365313654e-07,
+      "loss": 1.3319,
+      "step": 2018
+    },
+    {
+      "epoch": 2.51275668948351,
+      "grad_norm": 3.3614137172698975,
+      "learning_rate": 9.363468634686347e-07,
+      "loss": 1.383,
+      "step": 2019
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "grad_norm": 3.4617390632629395,
+      "learning_rate": 9.340405904059042e-07,
+      "loss": 1.3628,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5140012445550717,
+      "eval_loss": 1.6153606176376343,
+      "eval_runtime": 46.8889,
+      "eval_samples_per_second": 21.327,
+      "eval_steps_per_second": 0.896,
+      "step": 2020
+    },
+    {
+      "epoch": 2.5152457996266335,
+      "grad_norm": 3.3711016178131104,
+      "learning_rate": 9.317343173431735e-07,
+      "loss": 1.3449,
+      "step": 2021
+    },
+    {
+      "epoch": 2.5164903546981954,
+      "grad_norm": 3.2858471870422363,
+      "learning_rate": 9.294280442804429e-07,
+      "loss": 1.348,
+      "step": 2022
+    },
+    {
+      "epoch": 2.5177349097697572,
+      "grad_norm": 3.445166826248169,
+      "learning_rate": 9.271217712177122e-07,
+      "loss": 1.3182,
+      "step": 2023
+    },
+    {
+      "epoch": 2.518979464841319,
+      "grad_norm": 3.417206287384033,
+      "learning_rate": 9.248154981549816e-07,
+      "loss": 1.3241,
+      "step": 2024
+    },
+    {
+      "epoch": 2.520224019912881,
+      "grad_norm": 3.333965539932251,
+      "learning_rate": 9.22509225092251e-07,
+      "loss": 1.3214,
+      "step": 2025
+    },
+    {
+      "epoch": 2.521468574984443,
+      "grad_norm": 3.4090497493743896,
+      "learning_rate": 9.202029520295204e-07,
+      "loss": 1.3566,
+      "step": 2026
+    },
+    {
+      "epoch": 2.522713130056005,
+      "grad_norm": 3.312790870666504,
+      "learning_rate": 9.178966789667898e-07,
+      "loss": 1.3181,
+      "step": 2027
+    },
+    {
+      "epoch": 2.523957685127567,
+      "grad_norm": 3.254122734069824,
+      "learning_rate": 9.155904059040591e-07,
+      "loss": 1.3079,
+      "step": 2028
+    },
+    {
+      "epoch": 2.5252022401991288,
+      "grad_norm": 3.5144238471984863,
+      "learning_rate": 9.132841328413284e-07,
+      "loss": 1.3586,
+      "step": 2029
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "grad_norm": 3.6083173751831055,
+      "learning_rate": 9.109778597785979e-07,
+      "loss": 1.3833,
+      "step": 2030
+    },
+    {
+      "epoch": 2.5264467952706906,
+      "eval_loss": 1.6153796911239624,
+      "eval_runtime": 52.1312,
+      "eval_samples_per_second": 19.182,
+      "eval_steps_per_second": 0.806,
+      "step": 2030
+    },
+    {
+      "epoch": 2.527691350342253,
+      "grad_norm": 3.489710569381714,
+      "learning_rate": 9.086715867158673e-07,
+      "loss": 1.3466,
+      "step": 2031
+    },
+    {
+      "epoch": 2.5289359054138147,
+      "grad_norm": 3.4363298416137695,
+      "learning_rate": 9.063653136531365e-07,
+      "loss": 1.3461,
+      "step": 2032
+    },
+    {
+      "epoch": 2.5301804604853766,
+      "grad_norm": 3.627476930618286,
+      "learning_rate": 9.040590405904059e-07,
+      "loss": 1.3655,
+      "step": 2033
+    },
+    {
+      "epoch": 2.5314250155569384,
+      "grad_norm": 3.2995362281799316,
+      "learning_rate": 9.017527675276753e-07,
+      "loss": 1.297,
+      "step": 2034
+    },
+    {
+      "epoch": 2.5326695706285003,
+      "grad_norm": 3.4116721153259277,
+      "learning_rate": 8.994464944649448e-07,
+      "loss": 1.3319,
+      "step": 2035
+    },
+    {
+      "epoch": 2.533914125700062,
+      "grad_norm": 3.4598546028137207,
+      "learning_rate": 8.971402214022142e-07,
+      "loss": 1.3646,
+      "step": 2036
+    },
+    {
+      "epoch": 2.535158680771624,
+      "grad_norm": 3.4111695289611816,
+      "learning_rate": 8.948339483394834e-07,
+      "loss": 1.322,
+      "step": 2037
+    },
+    {
+      "epoch": 2.536403235843186,
+      "grad_norm": 3.371891736984253,
+      "learning_rate": 8.925276752767528e-07,
+      "loss": 1.362,
+      "step": 2038
+    },
+    {
+      "epoch": 2.537647790914748,
+      "grad_norm": 3.4267642498016357,
+      "learning_rate": 8.902214022140222e-07,
+      "loss": 1.3829,
+      "step": 2039
+    },
+    {
+      "epoch": 2.53889234598631,
+      "grad_norm": 3.2384777069091797,
+      "learning_rate": 8.879151291512917e-07,
+      "loss": 1.2771,
+      "step": 2040
+    },
+    {
+      "epoch": 2.53889234598631,
+      "eval_loss": 1.6134564876556396,
+      "eval_runtime": 49.4544,
+      "eval_samples_per_second": 20.221,
+      "eval_steps_per_second": 0.849,
+      "step": 2040
+    },
+    {
+      "epoch": 2.540136901057872,
+      "grad_norm": 3.3554086685180664,
+      "learning_rate": 8.856088560885609e-07,
+      "loss": 1.327,
+      "step": 2041
+    },
+    {
+      "epoch": 2.5413814561294337,
+      "grad_norm": 3.297962188720703,
+      "learning_rate": 8.833025830258303e-07,
+      "loss": 1.3257,
+      "step": 2042
+    },
+    {
+      "epoch": 2.5426260112009955,
+      "grad_norm": 3.4998221397399902,
+      "learning_rate": 8.809963099630997e-07,
+      "loss": 1.3782,
+      "step": 2043
+    },
+    {
+      "epoch": 2.543870566272558,
+      "grad_norm": 3.3661816120147705,
+      "learning_rate": 8.786900369003691e-07,
+      "loss": 1.3338,
+      "step": 2044
+    },
+    {
+      "epoch": 2.5451151213441197,
+      "grad_norm": 3.7328436374664307,
+      "learning_rate": 8.763837638376385e-07,
+      "loss": 1.4149,
+      "step": 2045
+    },
+    {
+      "epoch": 2.5463596764156815,
+      "grad_norm": 3.621159315109253,
+      "learning_rate": 8.740774907749078e-07,
+      "loss": 1.3104,
+      "step": 2046
+    },
+    {
+      "epoch": 2.5476042314872434,
+      "grad_norm": 3.5403215885162354,
+      "learning_rate": 8.717712177121772e-07,
+      "loss": 1.3278,
+      "step": 2047
+    },
+    {
+      "epoch": 2.548848786558805,
+      "grad_norm": 3.3124406337738037,
+      "learning_rate": 8.694649446494466e-07,
+      "loss": 1.3313,
+      "step": 2048
+    },
+    {
+      "epoch": 2.550093341630367,
+      "grad_norm": 3.4480574131011963,
+      "learning_rate": 8.671586715867159e-07,
+      "loss": 1.3225,
+      "step": 2049
+    },
+    {
+      "epoch": 2.551337896701929,
+      "grad_norm": 3.372326135635376,
+      "learning_rate": 8.648523985239853e-07,
+      "loss": 1.3377,
+      "step": 2050
+    },
+    {
+      "epoch": 2.551337896701929,
+      "eval_loss": 1.6056281328201294,
+      "eval_runtime": 52.0326,
+      "eval_samples_per_second": 19.219,
+      "eval_steps_per_second": 0.807,
+      "step": 2050
+    },
+    {
+      "epoch": 2.5525824517734907,
+      "grad_norm": 3.4119911193847656,
+      "learning_rate": 8.625461254612547e-07,
+      "loss": 1.3462,
+      "step": 2051
+    },
+    {
+      "epoch": 2.553827006845053,
+      "grad_norm": 3.5083751678466797,
+      "learning_rate": 8.602398523985241e-07,
+      "loss": 1.3632,
+      "step": 2052
+    },
+    {
+      "epoch": 2.555071561916615,
+      "grad_norm": 3.279911518096924,
+      "learning_rate": 8.579335793357934e-07,
+      "loss": 1.3326,
+      "step": 2053
+    },
+    {
+      "epoch": 2.5563161169881767,
+      "grad_norm": 3.2245192527770996,
+      "learning_rate": 8.556273062730628e-07,
+      "loss": 1.3033,
+      "step": 2054
+    },
+    {
+      "epoch": 2.5575606720597386,
+      "grad_norm": 3.3703253269195557,
+      "learning_rate": 8.533210332103322e-07,
+      "loss": 1.3219,
+      "step": 2055
+    },
+    {
+      "epoch": 2.5588052271313004,
+      "grad_norm": 3.3931314945220947,
+      "learning_rate": 8.510147601476016e-07,
+      "loss": 1.2992,
+      "step": 2056
+    },
+    {
+      "epoch": 2.5600497822028627,
+      "grad_norm": 3.5495870113372803,
+      "learning_rate": 8.487084870848709e-07,
+      "loss": 1.3872,
+      "step": 2057
+    },
+    {
+      "epoch": 2.5612943372744246,
+      "grad_norm": 3.476536273956299,
+      "learning_rate": 8.464022140221403e-07,
+      "loss": 1.3609,
+      "step": 2058
+    },
+    {
+      "epoch": 2.5625388923459864,
+      "grad_norm": 3.4503397941589355,
+      "learning_rate": 8.440959409594096e-07,
+      "loss": 1.339,
+      "step": 2059
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "grad_norm": 3.3848440647125244,
+      "learning_rate": 8.417896678966791e-07,
+      "loss": 1.3369,
+      "step": 2060
+    },
+    {
+      "epoch": 2.5637834474175483,
+      "eval_loss": 1.61435866355896,
+      "eval_runtime": 48.4317,
+      "eval_samples_per_second": 20.648,
+      "eval_steps_per_second": 0.867,
+      "step": 2060
+    },
+    {
+      "epoch": 2.56502800248911,
+      "grad_norm": 3.474571466445923,
+      "learning_rate": 8.394833948339484e-07,
+      "loss": 1.379,
+      "step": 2061
+    },
+    {
+      "epoch": 2.566272557560672,
+      "grad_norm": 3.487356185913086,
+      "learning_rate": 8.371771217712178e-07,
+      "loss": 1.365,
+      "step": 2062
+    },
+    {
+      "epoch": 2.567517112632234,
+      "grad_norm": 3.5649149417877197,
+      "learning_rate": 8.348708487084871e-07,
+      "loss": 1.3619,
+      "step": 2063
+    },
+    {
+      "epoch": 2.5687616677037957,
+      "grad_norm": 3.572577714920044,
+      "learning_rate": 8.325645756457565e-07,
+      "loss": 1.3877,
+      "step": 2064
+    },
+    {
+      "epoch": 2.570006222775358,
+      "grad_norm": 3.3802125453948975,
+      "learning_rate": 8.302583025830259e-07,
+      "loss": 1.3323,
+      "step": 2065
+    },
+    {
+      "epoch": 2.57125077784692,
+      "grad_norm": 3.317291498184204,
+      "learning_rate": 8.279520295202953e-07,
+      "loss": 1.3532,
+      "step": 2066
+    },
+    {
+      "epoch": 2.5724953329184816,
+      "grad_norm": 3.485429525375366,
+      "learning_rate": 8.256457564575647e-07,
+      "loss": 1.3561,
+      "step": 2067
+    },
+    {
+      "epoch": 2.5737398879900435,
+      "grad_norm": 3.4251649379730225,
+      "learning_rate": 8.233394833948339e-07,
+      "loss": 1.3494,
+      "step": 2068
+    },
+    {
+      "epoch": 2.5749844430616053,
+      "grad_norm": 3.3372538089752197,
+      "learning_rate": 8.210332103321033e-07,
+      "loss": 1.3116,
+      "step": 2069
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "grad_norm": 3.3535027503967285,
+      "learning_rate": 8.187269372693728e-07,
+      "loss": 1.3541,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5762289981331676,
+      "eval_loss": 1.6068458557128906,
+      "eval_runtime": 51.6237,
+      "eval_samples_per_second": 19.371,
+      "eval_steps_per_second": 0.814,
+      "step": 2070
+    },
+    {
+      "epoch": 2.5774735532047295,
+      "grad_norm": 3.4335787296295166,
+      "learning_rate": 8.164206642066422e-07,
+      "loss": 1.3884,
+      "step": 2071
+    },
+    {
+      "epoch": 2.5787181082762913,
+      "grad_norm": 3.436685800552368,
+      "learning_rate": 8.141143911439114e-07,
+      "loss": 1.3508,
+      "step": 2072
+    },
+    {
+      "epoch": 2.579962663347853,
+      "grad_norm": 3.333679676055908,
+      "learning_rate": 8.118081180811808e-07,
+      "loss": 1.3485,
+      "step": 2073
+    },
+    {
+      "epoch": 2.581207218419415,
+      "grad_norm": 3.4076309204101562,
+      "learning_rate": 8.095018450184502e-07,
+      "loss": 1.3219,
+      "step": 2074
+    },
+    {
+      "epoch": 2.582451773490977,
+      "grad_norm": 3.3851561546325684,
+      "learning_rate": 8.071955719557197e-07,
+      "loss": 1.3365,
+      "step": 2075
+    },
+    {
+      "epoch": 2.5836963285625387,
+      "grad_norm": 3.3714189529418945,
+      "learning_rate": 8.048892988929891e-07,
+      "loss": 1.3448,
+      "step": 2076
+    },
+    {
+      "epoch": 2.5849408836341006,
+      "grad_norm": 3.32585072517395,
+      "learning_rate": 8.025830258302583e-07,
+      "loss": 1.338,
+      "step": 2077
+    },
+    {
+      "epoch": 2.586185438705663,
+      "grad_norm": 3.4017553329467773,
+      "learning_rate": 8.002767527675277e-07,
+      "loss": 1.3597,
+      "step": 2078
+    },
+    {
+      "epoch": 2.5874299937772247,
+      "grad_norm": 3.334178924560547,
+      "learning_rate": 7.979704797047971e-07,
+      "loss": 1.2981,
+      "step": 2079
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "grad_norm": 3.443598985671997,
+      "learning_rate": 7.956642066420666e-07,
+      "loss": 1.3188,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5886745488487866,
+      "eval_loss": 1.6108036041259766,
+      "eval_runtime": 50.3752,
+      "eval_samples_per_second": 19.851,
+      "eval_steps_per_second": 0.834,
+      "step": 2080
+    },
+    {
+      "epoch": 2.5899191039203484,
+      "grad_norm": 3.460387706756592,
+      "learning_rate": 7.933579335793358e-07,
+      "loss": 1.3794,
+      "step": 2081
+    },
+    {
+      "epoch": 2.5911636589919103,
+      "grad_norm": 3.370560646057129,
+      "learning_rate": 7.910516605166052e-07,
+      "loss": 1.3068,
+      "step": 2082
+    },
+    {
+      "epoch": 2.5924082140634725,
+      "grad_norm": 3.55534291267395,
+      "learning_rate": 7.887453874538746e-07,
+      "loss": 1.3581,
+      "step": 2083
+    },
+    {
+      "epoch": 2.5936527691350344,
+      "grad_norm": 3.322735548019409,
+      "learning_rate": 7.86439114391144e-07,
+      "loss": 1.3399,
+      "step": 2084
+    },
+    {
+      "epoch": 2.5948973242065962,
+      "grad_norm": 3.5256693363189697,
+      "learning_rate": 7.841328413284134e-07,
+      "loss": 1.377,
+      "step": 2085
+    },
+    {
+      "epoch": 2.596141879278158,
+      "grad_norm": 3.4358303546905518,
+      "learning_rate": 7.818265682656827e-07,
+      "loss": 1.3486,
+      "step": 2086
+    },
+    {
+      "epoch": 2.59738643434972,
+      "grad_norm": 3.401855230331421,
+      "learning_rate": 7.795202952029521e-07,
+      "loss": 1.3013,
+      "step": 2087
+    },
+    {
+      "epoch": 2.598630989421282,
+      "grad_norm": 3.414670467376709,
+      "learning_rate": 7.772140221402215e-07,
+      "loss": 1.3342,
+      "step": 2088
+    },
+    {
+      "epoch": 2.5998755444928436,
+      "grad_norm": 3.3348028659820557,
+      "learning_rate": 7.749077490774908e-07,
+      "loss": 1.3594,
+      "step": 2089
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "grad_norm": 3.4095754623413086,
+      "learning_rate": 7.726014760147602e-07,
+      "loss": 1.3739,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6011200995644055,
+      "eval_loss": 1.6106606721878052,
+      "eval_runtime": 49.3878,
+      "eval_samples_per_second": 20.248,
+      "eval_steps_per_second": 0.85,
+      "step": 2090
+    },
+    {
+      "epoch": 2.6023646546359678,
+      "grad_norm": 3.247753143310547,
+      "learning_rate": 7.702952029520296e-07,
+      "loss": 1.3026,
+      "step": 2091
+    },
+    {
+      "epoch": 2.6036092097075296,
+      "grad_norm": 3.3973324298858643,
+      "learning_rate": 7.67988929889299e-07,
+      "loss": 1.3381,
+      "step": 2092
+    },
+    {
+      "epoch": 2.6048537647790915,
+      "grad_norm": 3.4262280464172363,
+      "learning_rate": 7.656826568265683e-07,
+      "loss": 1.3595,
+      "step": 2093
+    },
+    {
+      "epoch": 2.6060983198506533,
+      "grad_norm": 3.401784658432007,
+      "learning_rate": 7.633763837638377e-07,
+      "loss": 1.3339,
+      "step": 2094
+    },
+    {
+      "epoch": 2.607342874922215,
+      "grad_norm": 3.3231465816497803,
+      "learning_rate": 7.610701107011071e-07,
+      "loss": 1.2529,
+      "step": 2095
+    },
+    {
+      "epoch": 2.6085874299937775,
+      "grad_norm": 3.412613868713379,
+      "learning_rate": 7.587638376383765e-07,
+      "loss": 1.3341,
+      "step": 2096
+    },
+    {
+      "epoch": 2.6098319850653393,
+      "grad_norm": 3.456622362136841,
+      "learning_rate": 7.564575645756458e-07,
+      "loss": 1.3545,
+      "step": 2097
+    },
+    {
+      "epoch": 2.611076540136901,
+      "grad_norm": 3.488546133041382,
+      "learning_rate": 7.541512915129152e-07,
+      "loss": 1.3405,
+      "step": 2098
+    },
+    {
+      "epoch": 2.612321095208463,
+      "grad_norm": 3.4436495304107666,
+      "learning_rate": 7.518450184501845e-07,
+      "loss": 1.3242,
+      "step": 2099
+    },
+    {
+      "epoch": 2.613565650280025,
+      "grad_norm": 3.382150411605835,
+      "learning_rate": 7.49538745387454e-07,
+      "loss": 1.301,
+      "step": 2100
+    },
+    {
+      "epoch": 2.613565650280025,
+      "eval_loss": 1.6055735349655151,
+      "eval_runtime": 48.001,
+      "eval_samples_per_second": 20.833,
+      "eval_steps_per_second": 0.875,
+      "step": 2100
+    },
+    {
+      "epoch": 2.6148102053515867,
+      "grad_norm": 3.478010892868042,
+      "learning_rate": 7.472324723247233e-07,
+      "loss": 1.3565,
+      "step": 2101
+    },
+    {
+      "epoch": 2.6160547604231486,
+      "grad_norm": 3.4550209045410156,
+      "learning_rate": 7.449261992619927e-07,
+      "loss": 1.3732,
+      "step": 2102
+    },
+    {
+      "epoch": 2.6172993154947104,
+      "grad_norm": 3.3357841968536377,
+      "learning_rate": 7.42619926199262e-07,
+      "loss": 1.3072,
+      "step": 2103
+    },
+    {
+      "epoch": 2.6185438705662727,
+      "grad_norm": 3.4329988956451416,
+      "learning_rate": 7.403136531365313e-07,
+      "loss": 1.3353,
+      "step": 2104
+    },
+    {
+      "epoch": 2.6197884256378345,
+      "grad_norm": 3.390272378921509,
+      "learning_rate": 7.380073800738008e-07,
+      "loss": 1.3537,
+      "step": 2105
+    },
+    {
+      "epoch": 2.6210329807093964,
+      "grad_norm": 3.4377708435058594,
+      "learning_rate": 7.357011070110702e-07,
+      "loss": 1.357,
+      "step": 2106
+    },
+    {
+      "epoch": 2.6222775357809582,
+      "grad_norm": 3.383354425430298,
+      "learning_rate": 7.333948339483396e-07,
+      "loss": 1.3663,
+      "step": 2107
+    },
+    {
+      "epoch": 2.62352209085252,
+      "grad_norm": 3.364445209503174,
+      "learning_rate": 7.310885608856088e-07,
+      "loss": 1.3515,
+      "step": 2108
+    },
+    {
+      "epoch": 2.6247666459240824,
+      "grad_norm": 3.357224702835083,
+      "learning_rate": 7.287822878228782e-07,
+      "loss": 1.3436,
+      "step": 2109
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "grad_norm": 3.340733289718628,
+      "learning_rate": 7.264760147601477e-07,
+      "loss": 1.3481,
+      "step": 2110
+    },
+    {
+      "epoch": 2.6260112009956442,
+      "eval_loss": 1.609564185142517,
+      "eval_runtime": 43.5251,
+      "eval_samples_per_second": 22.975,
+      "eval_steps_per_second": 0.965,
+      "step": 2110
+    },
+    {
+      "epoch": 2.627255756067206,
+      "grad_norm": 3.307832717895508,
+      "learning_rate": 7.241697416974171e-07,
+      "loss": 1.322,
+      "step": 2111
+    },
+    {
+      "epoch": 2.628500311138768,
+      "grad_norm": 3.3134219646453857,
+      "learning_rate": 7.218634686346863e-07,
+      "loss": 1.2888,
+      "step": 2112
+    },
+    {
+      "epoch": 2.6297448662103298,
+      "grad_norm": 3.5507848262786865,
+      "learning_rate": 7.195571955719557e-07,
+      "loss": 1.3986,
+      "step": 2113
+    },
+    {
+      "epoch": 2.6309894212818916,
+      "grad_norm": 3.26234769821167,
+      "learning_rate": 7.172509225092251e-07,
+      "loss": 1.3037,
+      "step": 2114
+    },
+    {
+      "epoch": 2.6322339763534535,
+      "grad_norm": 3.4239728450775146,
+      "learning_rate": 7.149446494464946e-07,
+      "loss": 1.326,
+      "step": 2115
+    },
+    {
+      "epoch": 2.6334785314250153,
+      "grad_norm": 3.470409870147705,
+      "learning_rate": 7.12638376383764e-07,
+      "loss": 1.3435,
+      "step": 2116
+    },
+    {
+      "epoch": 2.6347230864965776,
+      "grad_norm": 3.4207205772399902,
+      "learning_rate": 7.103321033210332e-07,
+      "loss": 1.3519,
+      "step": 2117
+    },
+    {
+      "epoch": 2.6359676415681395,
+      "grad_norm": 3.3702151775360107,
+      "learning_rate": 7.080258302583026e-07,
+      "loss": 1.3081,
+      "step": 2118
+    },
+    {
+      "epoch": 2.6372121966397013,
+      "grad_norm": 3.406705379486084,
+      "learning_rate": 7.05719557195572e-07,
+      "loss": 1.3698,
+      "step": 2119
+    },
+    {
+      "epoch": 2.638456751711263,
+      "grad_norm": 3.5395123958587646,
+      "learning_rate": 7.034132841328415e-07,
+      "loss": 1.3543,
+      "step": 2120
+    },
+    {
+      "epoch": 2.638456751711263,
+      "eval_loss": 1.6064285039901733,
+      "eval_runtime": 47.1124,
+      "eval_samples_per_second": 21.226,
+      "eval_steps_per_second": 0.891,
+      "step": 2120
+    },
+    {
+      "epoch": 2.639701306782825,
+      "grad_norm": 3.3317949771881104,
+      "learning_rate": 7.011070110701107e-07,
+      "loss": 1.3401,
+      "step": 2121
+    },
+    {
+      "epoch": 2.6409458618543873,
+      "grad_norm": 3.3046767711639404,
+      "learning_rate": 6.988007380073801e-07,
+      "loss": 1.355,
+      "step": 2122
+    },
+    {
+      "epoch": 2.642190416925949,
+      "grad_norm": 3.419886350631714,
+      "learning_rate": 6.964944649446495e-07,
+      "loss": 1.2955,
+      "step": 2123
+    },
+    {
+      "epoch": 2.643434971997511,
+      "grad_norm": 3.3755574226379395,
+      "learning_rate": 6.94188191881919e-07,
+      "loss": 1.3093,
+      "step": 2124
+    },
+    {
+      "epoch": 2.644679527069073,
+      "grad_norm": 3.490931510925293,
+      "learning_rate": 6.918819188191883e-07,
+      "loss": 1.3994,
+      "step": 2125
+    },
+    {
+      "epoch": 2.6459240821406347,
+      "grad_norm": 3.48911452293396,
+      "learning_rate": 6.895756457564576e-07,
+      "loss": 1.3449,
+      "step": 2126
+    },
+    {
+      "epoch": 2.6471686372121965,
+      "grad_norm": 3.5786304473876953,
+      "learning_rate": 6.87269372693727e-07,
+      "loss": 1.3569,
+      "step": 2127
+    },
+    {
+      "epoch": 2.6484131922837584,
+      "grad_norm": 3.500648260116577,
+      "learning_rate": 6.849630996309964e-07,
+      "loss": 1.349,
+      "step": 2128
+    },
+    {
+      "epoch": 2.6496577473553202,
+      "grad_norm": 3.644001007080078,
+      "learning_rate": 6.826568265682657e-07,
+      "loss": 1.3534,
+      "step": 2129
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "grad_norm": 3.3233609199523926,
+      "learning_rate": 6.803505535055351e-07,
+      "loss": 1.303,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6509023024268825,
+      "eval_loss": 1.6085610389709473,
+      "eval_runtime": 51.443,
+      "eval_samples_per_second": 19.439,
+      "eval_steps_per_second": 0.816,
+      "step": 2130
+    },
+    {
+      "epoch": 2.6521468574984444,
+      "grad_norm": 3.413889169692993,
+      "learning_rate": 6.780442804428045e-07,
+      "loss": 1.335,
+      "step": 2131
+    },
+    {
+      "epoch": 2.653391412570006,
+      "grad_norm": 3.417112112045288,
+      "learning_rate": 6.757380073800739e-07,
+      "loss": 1.3768,
+      "step": 2132
+    },
+    {
+      "epoch": 2.654635967641568,
+      "grad_norm": 3.4491703510284424,
+      "learning_rate": 6.734317343173432e-07,
+      "loss": 1.3493,
+      "step": 2133
+    },
+    {
+      "epoch": 2.65588052271313,
+      "grad_norm": 3.3965368270874023,
+      "learning_rate": 6.711254612546127e-07,
+      "loss": 1.3328,
+      "step": 2134
+    },
+    {
+      "epoch": 2.657125077784692,
+      "grad_norm": 3.426530361175537,
+      "learning_rate": 6.68819188191882e-07,
+      "loss": 1.3548,
+      "step": 2135
+    },
+    {
+      "epoch": 2.658369632856254,
+      "grad_norm": 3.386624813079834,
+      "learning_rate": 6.665129151291514e-07,
+      "loss": 1.3523,
+      "step": 2136
+    },
+    {
+      "epoch": 2.659614187927816,
+      "grad_norm": 3.383504867553711,
+      "learning_rate": 6.642066420664207e-07,
+      "loss": 1.3356,
+      "step": 2137
+    },
+    {
+      "epoch": 2.6608587429993777,
+      "grad_norm": 3.3986196517944336,
+      "learning_rate": 6.619003690036901e-07,
+      "loss": 1.3605,
+      "step": 2138
+    },
+    {
+      "epoch": 2.6621032980709396,
+      "grad_norm": 3.4832820892333984,
+      "learning_rate": 6.595940959409594e-07,
+      "loss": 1.395,
+      "step": 2139
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "grad_norm": 3.498077154159546,
+      "learning_rate": 6.572878228782288e-07,
+      "loss": 1.347,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6633478531425014,
+      "eval_loss": 1.6076886653900146,
+      "eval_runtime": 47.7587,
+      "eval_samples_per_second": 20.939,
+      "eval_steps_per_second": 0.879,
+      "step": 2140
+    },
+    {
+      "epoch": 2.6645924082140633,
+      "grad_norm": 3.6382765769958496,
+      "learning_rate": 6.549815498154982e-07,
+      "loss": 1.4288,
+      "step": 2141
+    },
+    {
+      "epoch": 2.665836963285625,
+      "grad_norm": 3.434046983718872,
+      "learning_rate": 6.526752767527676e-07,
+      "loss": 1.3258,
+      "step": 2142
+    },
+    {
+      "epoch": 2.6670815183571874,
+      "grad_norm": 3.381760358810425,
+      "learning_rate": 6.503690036900369e-07,
+      "loss": 1.3567,
+      "step": 2143
+    },
+    {
+      "epoch": 2.6683260734287493,
+      "grad_norm": 3.450314521789551,
+      "learning_rate": 6.480627306273062e-07,
+      "loss": 1.3504,
+      "step": 2144
+    },
+    {
+      "epoch": 2.669570628500311,
+      "grad_norm": 3.337059736251831,
+      "learning_rate": 6.457564575645757e-07,
+      "loss": 1.3664,
+      "step": 2145
+    },
+    {
+      "epoch": 2.670815183571873,
+      "grad_norm": 3.415597438812256,
+      "learning_rate": 6.434501845018451e-07,
+      "loss": 1.369,
+      "step": 2146
+    },
+    {
+      "epoch": 2.672059738643435,
+      "grad_norm": 3.331700325012207,
+      "learning_rate": 6.411439114391145e-07,
+      "loss": 1.3056,
+      "step": 2147
+    },
+    {
+      "epoch": 2.673304293714997,
+      "grad_norm": 3.439768075942993,
+      "learning_rate": 6.388376383763837e-07,
+      "loss": 1.3591,
+      "step": 2148
+    },
+    {
+      "epoch": 2.674548848786559,
+      "grad_norm": 3.4256532192230225,
+      "learning_rate": 6.365313653136531e-07,
+      "loss": 1.3581,
+      "step": 2149
+    },
+    {
+      "epoch": 2.675793403858121,
+      "grad_norm": 3.337472438812256,
+      "learning_rate": 6.342250922509226e-07,
+      "loss": 1.358,
+      "step": 2150
+    },
+    {
+      "epoch": 2.675793403858121,
+      "eval_loss": 1.6052860021591187,
+      "eval_runtime": 51.6432,
+      "eval_samples_per_second": 19.364,
+      "eval_steps_per_second": 0.813,
+      "step": 2150
+    },
+    {
+      "epoch": 2.6770379589296827,
+      "grad_norm": 3.458285331726074,
+      "learning_rate": 6.31918819188192e-07,
+      "loss": 1.346,
+      "step": 2151
+    },
+    {
+      "epoch": 2.6782825140012445,
+      "grad_norm": 3.3939685821533203,
+      "learning_rate": 6.296125461254612e-07,
+      "loss": 1.3634,
+      "step": 2152
+    },
+    {
+      "epoch": 2.6795270690728064,
+      "grad_norm": 3.4758291244506836,
+      "learning_rate": 6.273062730627306e-07,
+      "loss": 1.3913,
+      "step": 2153
+    },
+    {
+      "epoch": 2.680771624144368,
+      "grad_norm": 3.316906690597534,
+      "learning_rate": 6.25e-07,
+      "loss": 1.2944,
+      "step": 2154
+    },
+    {
+      "epoch": 2.68201617921593,
+      "grad_norm": 3.357058048248291,
+      "learning_rate": 6.226937269372695e-07,
+      "loss": 1.321,
+      "step": 2155
+    },
+    {
+      "epoch": 2.6832607342874923,
+      "grad_norm": 3.4547276496887207,
+      "learning_rate": 6.203874538745388e-07,
+      "loss": 1.3407,
+      "step": 2156
+    },
+    {
+      "epoch": 2.684505289359054,
+      "grad_norm": 3.3258347511291504,
+      "learning_rate": 6.180811808118082e-07,
+      "loss": 1.3526,
+      "step": 2157
+    },
+    {
+      "epoch": 2.685749844430616,
+      "grad_norm": 3.418070077896118,
+      "learning_rate": 6.157749077490775e-07,
+      "loss": 1.3764,
+      "step": 2158
+    },
+    {
+      "epoch": 2.686994399502178,
+      "grad_norm": 3.2903330326080322,
+      "learning_rate": 6.13468634686347e-07,
+      "loss": 1.3144,
+      "step": 2159
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "grad_norm": 3.340200901031494,
+      "learning_rate": 6.111623616236163e-07,
+      "loss": 1.3399,
+      "step": 2160
+    },
+    {
+      "epoch": 2.6882389545737397,
+      "eval_loss": 1.6079058647155762,
+      "eval_runtime": 52.2924,
+      "eval_samples_per_second": 19.123,
+      "eval_steps_per_second": 0.803,
+      "step": 2160
+    },
+    {
+      "epoch": 2.689483509645302,
+      "grad_norm": 3.4739818572998047,
+      "learning_rate": 6.088560885608857e-07,
+      "loss": 1.333,
+      "step": 2161
+    },
+    {
+      "epoch": 2.690728064716864,
+      "grad_norm": 3.5063717365264893,
+      "learning_rate": 6.06549815498155e-07,
+      "loss": 1.3089,
+      "step": 2162
+    },
+    {
+      "epoch": 2.6919726197884257,
+      "grad_norm": 3.34189510345459,
+      "learning_rate": 6.042435424354244e-07,
+      "loss": 1.3173,
+      "step": 2163
+    },
+    {
+      "epoch": 2.6932171748599876,
+      "grad_norm": 3.385422706604004,
+      "learning_rate": 6.019372693726938e-07,
+      "loss": 1.3605,
+      "step": 2164
+    },
+    {
+      "epoch": 2.6944617299315494,
+      "grad_norm": 3.3808515071868896,
+      "learning_rate": 5.996309963099632e-07,
+      "loss": 1.3341,
+      "step": 2165
+    },
+    {
+      "epoch": 2.6957062850031113,
+      "grad_norm": 3.546069383621216,
+      "learning_rate": 5.973247232472326e-07,
+      "loss": 1.3706,
+      "step": 2166
+    },
+    {
+      "epoch": 2.696950840074673,
+      "grad_norm": 3.3705339431762695,
+      "learning_rate": 5.950184501845019e-07,
+      "loss": 1.3226,
+      "step": 2167
+    },
+    {
+      "epoch": 2.698195395146235,
+      "grad_norm": 3.4087791442871094,
+      "learning_rate": 5.927121771217713e-07,
+      "loss": 1.3285,
+      "step": 2168
+    },
+    {
+      "epoch": 2.6994399502177973,
+      "grad_norm": 3.2774477005004883,
+      "learning_rate": 5.904059040590407e-07,
+      "loss": 1.3128,
+      "step": 2169
+    },
+    {
+      "epoch": 2.700684505289359,
+      "grad_norm": 3.3653030395507812,
+      "learning_rate": 5.880996309963101e-07,
+      "loss": 1.3395,
+      "step": 2170
+    },
+    {
+      "epoch": 2.700684505289359,
+      "eval_loss": 1.608220100402832,
+      "eval_runtime": 53.3978,
+      "eval_samples_per_second": 18.727,
+      "eval_steps_per_second": 0.787,
+      "step": 2170
+    },
+    {
+      "epoch": 2.701929060360921,
+      "grad_norm": 3.394524335861206,
+      "learning_rate": 5.857933579335794e-07,
+      "loss": 1.2853,
+      "step": 2171
+    },
+    {
+      "epoch": 2.703173615432483,
+      "grad_norm": 3.3898823261260986,
+      "learning_rate": 5.834870848708487e-07,
+      "loss": 1.3494,
+      "step": 2172
+    },
+    {
+      "epoch": 2.7044181705040446,
+      "grad_norm": 3.394688367843628,
+      "learning_rate": 5.811808118081181e-07,
+      "loss": 1.3129,
+      "step": 2173
+    },
+    {
+      "epoch": 2.705662725575607,
+      "grad_norm": 3.4164483547210693,
+      "learning_rate": 5.788745387453875e-07,
+      "loss": 1.368,
+      "step": 2174
+    },
+    {
+      "epoch": 2.706907280647169,
+      "grad_norm": 3.2976245880126953,
+      "learning_rate": 5.765682656826569e-07,
+      "loss": 1.361,
+      "step": 2175
+    },
+    {
+      "epoch": 2.7081518357187306,
+      "grad_norm": 3.307027816772461,
+      "learning_rate": 5.742619926199262e-07,
+      "loss": 1.3509,
+      "step": 2176
+    },
+    {
+      "epoch": 2.7093963907902925,
+      "grad_norm": 3.4393889904022217,
+      "learning_rate": 5.719557195571956e-07,
+      "loss": 1.331,
+      "step": 2177
+    },
+    {
+      "epoch": 2.7106409458618543,
+      "grad_norm": 3.312872886657715,
+      "learning_rate": 5.696494464944649e-07,
+      "loss": 1.3541,
+      "step": 2178
+    },
+    {
+      "epoch": 2.711885500933416,
+      "grad_norm": 3.4782755374908447,
+      "learning_rate": 5.673431734317344e-07,
+      "loss": 1.3078,
+      "step": 2179
+    },
+    {
+      "epoch": 2.713130056004978,
+      "grad_norm": 3.457902193069458,
+      "learning_rate": 5.650369003690037e-07,
+      "loss": 1.3712,
+      "step": 2180
+    },
+    {
+      "epoch": 2.713130056004978,
+      "eval_loss": 1.6061460971832275,
+      "eval_runtime": 50.4797,
+      "eval_samples_per_second": 19.81,
+      "eval_steps_per_second": 0.832,
+      "step": 2180
+    },
+    {
+      "epoch": 2.71437461107654,
+      "grad_norm": 3.508249282836914,
+      "learning_rate": 5.627306273062731e-07,
+      "loss": 1.3948,
+      "step": 2181
+    },
+    {
+      "epoch": 2.715619166148102,
+      "grad_norm": 3.3043899536132812,
+      "learning_rate": 5.604243542435424e-07,
+      "loss": 1.3382,
+      "step": 2182
+    },
+    {
+      "epoch": 2.716863721219664,
+      "grad_norm": 3.4448275566101074,
+      "learning_rate": 5.581180811808118e-07,
+      "loss": 1.3422,
+      "step": 2183
+    },
+    {
+      "epoch": 2.718108276291226,
+      "grad_norm": 3.3538706302642822,
+      "learning_rate": 5.558118081180812e-07,
+      "loss": 1.3054,
+      "step": 2184
+    },
+    {
+      "epoch": 2.7193528313627877,
+      "grad_norm": 3.4720618724823,
+      "learning_rate": 5.535055350553506e-07,
+      "loss": 1.3525,
+      "step": 2185
+    },
+    {
+      "epoch": 2.7205973864343496,
+      "grad_norm": 3.370326519012451,
+      "learning_rate": 5.5119926199262e-07,
+      "loss": 1.3756,
+      "step": 2186
+    },
+    {
+      "epoch": 2.721841941505912,
+      "grad_norm": 3.4985008239746094,
+      "learning_rate": 5.488929889298893e-07,
+      "loss": 1.3509,
+      "step": 2187
+    },
+    {
+      "epoch": 2.7230864965774737,
+      "grad_norm": 3.3014514446258545,
+      "learning_rate": 5.465867158671587e-07,
+      "loss": 1.2928,
+      "step": 2188
+    },
+    {
+      "epoch": 2.7243310516490356,
+      "grad_norm": 3.491454839706421,
+      "learning_rate": 5.44280442804428e-07,
+      "loss": 1.315,
+      "step": 2189
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "grad_norm": 3.3876538276672363,
+      "learning_rate": 5.419741697416975e-07,
+      "loss": 1.3344,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7255756067205974,
+      "eval_loss": 1.606774926185608,
+      "eval_runtime": 49.375,
+      "eval_samples_per_second": 20.253,
+      "eval_steps_per_second": 0.851,
+      "step": 2190
+    },
+    {
+      "epoch": 2.7268201617921592,
+      "grad_norm": 3.2686572074890137,
+      "learning_rate": 5.396678966789668e-07,
+      "loss": 1.3334,
+      "step": 2191
+    },
+    {
+      "epoch": 2.728064716863721,
+      "grad_norm": 3.539928913116455,
+      "learning_rate": 5.373616236162362e-07,
+      "loss": 1.3676,
+      "step": 2192
+    },
+    {
+      "epoch": 2.729309271935283,
+      "grad_norm": 3.424800395965576,
+      "learning_rate": 5.350553505535055e-07,
+      "loss": 1.3207,
+      "step": 2193
+    },
+    {
+      "epoch": 2.730553827006845,
+      "grad_norm": 3.459237575531006,
+      "learning_rate": 5.32749077490775e-07,
+      "loss": 1.3459,
+      "step": 2194
+    },
+    {
+      "epoch": 2.731798382078407,
+      "grad_norm": 3.39109468460083,
+      "learning_rate": 5.304428044280444e-07,
+      "loss": 1.3408,
+      "step": 2195
+    },
+    {
+      "epoch": 2.733042937149969,
+      "grad_norm": 3.4368112087249756,
+      "learning_rate": 5.281365313653137e-07,
+      "loss": 1.2679,
+      "step": 2196
+    },
+    {
+      "epoch": 2.734287492221531,
+      "grad_norm": 3.3579020500183105,
+      "learning_rate": 5.258302583025831e-07,
+      "loss": 1.3531,
+      "step": 2197
+    },
+    {
+      "epoch": 2.7355320472930926,
+      "grad_norm": 3.2895281314849854,
+      "learning_rate": 5.235239852398524e-07,
+      "loss": 1.3127,
+      "step": 2198
+    },
+    {
+      "epoch": 2.7367766023646545,
+      "grad_norm": 3.546879529953003,
+      "learning_rate": 5.212177121771219e-07,
+      "loss": 1.3816,
+      "step": 2199
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "grad_norm": 3.4015355110168457,
+      "learning_rate": 5.189114391143912e-07,
+      "loss": 1.3026,
+      "step": 2200
+    },
+    {
+      "epoch": 2.7380211574362168,
+      "eval_loss": 1.6100414991378784,
+      "eval_runtime": 44.238,
+      "eval_samples_per_second": 22.605,
+      "eval_steps_per_second": 0.949,
+      "step": 2200
+    },
+    {
+      "epoch": 2.7392657125077786,
+      "grad_norm": 3.4278082847595215,
+      "learning_rate": 5.166051660516606e-07,
+      "loss": 1.3545,
+      "step": 2201
+    },
+    {
+      "epoch": 2.7405102675793405,
+      "grad_norm": 3.4757940769195557,
+      "learning_rate": 5.142988929889299e-07,
+      "loss": 1.3632,
+      "step": 2202
+    },
+    {
+      "epoch": 2.7417548226509023,
+      "grad_norm": 3.419520854949951,
+      "learning_rate": 5.119926199261993e-07,
+      "loss": 1.3591,
+      "step": 2203
+    },
+    {
+      "epoch": 2.742999377722464,
+      "grad_norm": 3.348172903060913,
+      "learning_rate": 5.096863468634687e-07,
+      "loss": 1.3218,
+      "step": 2204
+    },
+    {
+      "epoch": 2.744243932794026,
+      "grad_norm": 3.4806814193725586,
+      "learning_rate": 5.073800738007381e-07,
+      "loss": 1.3907,
+      "step": 2205
+    },
+    {
+      "epoch": 2.745488487865588,
+      "grad_norm": 3.465930700302124,
+      "learning_rate": 5.050738007380075e-07,
+      "loss": 1.3458,
+      "step": 2206
+    },
+    {
+      "epoch": 2.7467330429371497,
+      "grad_norm": 3.377190351486206,
+      "learning_rate": 5.027675276752768e-07,
+      "loss": 1.3265,
+      "step": 2207
+    },
+    {
+      "epoch": 2.747977598008712,
+      "grad_norm": 3.2912864685058594,
+      "learning_rate": 5.004612546125461e-07,
+      "loss": 1.3413,
+      "step": 2208
+    },
+    {
+      "epoch": 2.749222153080274,
+      "grad_norm": 3.4280965328216553,
+      "learning_rate": 4.981549815498156e-07,
+      "loss": 1.3274,
+      "step": 2209
+    },
+    {
+      "epoch": 2.7504667081518357,
+      "grad_norm": 3.232537031173706,
+      "learning_rate": 4.958487084870849e-07,
+      "loss": 1.2905,
+      "step": 2210
+    },
+    {
+      "epoch": 2.7504667081518357,
+      "eval_loss": 1.6082451343536377,
+      "eval_runtime": 41.3326,
+      "eval_samples_per_second": 24.194,
+      "eval_steps_per_second": 1.016,
+      "step": 2210
+    },
+    {
+      "epoch": 2.7517112632233975,
+      "grad_norm": 3.5405726432800293,
+      "learning_rate": 4.935424354243543e-07,
+      "loss": 1.3306,
+      "step": 2211
+    },
+    {
+      "epoch": 2.7529558182949594,
+      "grad_norm": 3.414891481399536,
+      "learning_rate": 4.912361623616236e-07,
+      "loss": 1.3425,
+      "step": 2212
+    },
+    {
+      "epoch": 2.7542003733665217,
+      "grad_norm": 3.4994254112243652,
+      "learning_rate": 4.88929889298893e-07,
+      "loss": 1.3429,
+      "step": 2213
+    },
+    {
+      "epoch": 2.7554449284380835,
+      "grad_norm": 3.3640542030334473,
+      "learning_rate": 4.866236162361624e-07,
+      "loss": 1.3247,
+      "step": 2214
+    },
+    {
+      "epoch": 2.7566894835096454,
+      "grad_norm": 3.4134693145751953,
+      "learning_rate": 4.843173431734318e-07,
+      "loss": 1.3437,
+      "step": 2215
+    },
+    {
+      "epoch": 2.7579340385812072,
+      "grad_norm": 3.4103310108184814,
+      "learning_rate": 4.820110701107011e-07,
+      "loss": 1.3612,
+      "step": 2216
+    },
+    {
+      "epoch": 2.759178593652769,
+      "grad_norm": 3.3797495365142822,
+      "learning_rate": 4.797047970479705e-07,
+      "loss": 1.3379,
+      "step": 2217
+    },
+    {
+      "epoch": 2.760423148724331,
+      "grad_norm": 3.40238094329834,
+      "learning_rate": 4.773985239852398e-07,
+      "loss": 1.3301,
+      "step": 2218
+    },
+    {
+      "epoch": 2.7616677037958928,
+      "grad_norm": 3.350519895553589,
+      "learning_rate": 4.7509225092250926e-07,
+      "loss": 1.3228,
+      "step": 2219
+    },
+    {
+      "epoch": 2.7629122588674546,
+      "grad_norm": 3.2626607418060303,
+      "learning_rate": 4.7278597785977863e-07,
+      "loss": 1.3008,
+      "step": 2220
+    },
+    {
+      "epoch": 2.7629122588674546,
+      "eval_loss": 1.6074291467666626,
+      "eval_runtime": 43.4661,
+      "eval_samples_per_second": 23.006,
+      "eval_steps_per_second": 0.966,
+      "step": 2220
+    },
+    {
+      "epoch": 2.764156813939017,
+      "grad_norm": 3.508967161178589,
+      "learning_rate": 4.70479704797048e-07,
+      "loss": 1.3492,
+      "step": 2221
+    },
+    {
+      "epoch": 2.7654013690105788,
+      "grad_norm": 3.5330708026885986,
+      "learning_rate": 4.6817343173431737e-07,
+      "loss": 1.395,
+      "step": 2222
+    },
+    {
+      "epoch": 2.7666459240821406,
+      "grad_norm": 3.3957931995391846,
+      "learning_rate": 4.6586715867158674e-07,
+      "loss": 1.3331,
+      "step": 2223
+    },
+    {
+      "epoch": 2.7678904791537025,
+      "grad_norm": 3.3957931995391846,
+      "learning_rate": 4.6586715867158674e-07,
+      "loss": 1.3279,
+      "step": 2224
+    },
+    {
+      "epoch": 2.7691350342252643,
+      "grad_norm": 3.4120025634765625,
+      "learning_rate": 4.635608856088561e-07,
+      "loss": 1.3445,
+      "step": 2225
+    },
+    {
+      "epoch": 2.7703795892968266,
+      "grad_norm": 3.4226207733154297,
+      "learning_rate": 4.612546125461255e-07,
+      "loss": 1.2962,
+      "step": 2226
+    },
+    {
+      "epoch": 2.7716241443683884,
+      "grad_norm": 3.4244253635406494,
+      "learning_rate": 4.589483394833949e-07,
+      "loss": 1.3615,
+      "step": 2227
+    },
+    {
+      "epoch": 2.7728686994399503,
+      "grad_norm": 3.4812207221984863,
+      "learning_rate": 4.566420664206642e-07,
+      "loss": 1.3881,
+      "step": 2228
+    },
+    {
+      "epoch": 2.774113254511512,
+      "grad_norm": 3.3681764602661133,
+      "learning_rate": 4.5433579335793364e-07,
+      "loss": 1.3652,
+      "step": 2229
+    },
+    {
+      "epoch": 2.775357809583074,
+      "grad_norm": 3.3974297046661377,
+      "learning_rate": 4.5202952029520296e-07,
+      "loss": 1.303,
+      "step": 2230
+    },
+    {
+      "epoch": 2.775357809583074,
+      "eval_loss": 1.6103951930999756,
+      "eval_runtime": 44.0184,
+      "eval_samples_per_second": 22.718,
+      "eval_steps_per_second": 0.954,
+      "step": 2230
+    },
+    {
+      "epoch": 2.776602364654636,
+      "grad_norm": 3.2922801971435547,
+      "learning_rate": 4.497232472324724e-07,
+      "loss": 1.3118,
+      "step": 2231
+    },
+    {
+      "epoch": 2.7778469197261977,
+      "grad_norm": 3.332679271697998,
+      "learning_rate": 4.474169741697417e-07,
+      "loss": 1.3506,
+      "step": 2232
+    },
+    {
+      "epoch": 2.7790914747977595,
+      "grad_norm": 3.288560628890991,
+      "learning_rate": 4.451107011070111e-07,
+      "loss": 1.3231,
+      "step": 2233
+    },
+    {
+      "epoch": 2.780336029869322,
+      "grad_norm": 3.3787007331848145,
+      "learning_rate": 4.4280442804428044e-07,
+      "loss": 1.3377,
+      "step": 2234
+    },
+    {
+      "epoch": 2.7815805849408837,
+      "grad_norm": 3.529020071029663,
+      "learning_rate": 4.4049815498154986e-07,
+      "loss": 1.3268,
+      "step": 2235
+    },
+    {
+      "epoch": 2.7828251400124455,
+      "grad_norm": 3.3797807693481445,
+      "learning_rate": 4.3819188191881923e-07,
+      "loss": 1.3024,
+      "step": 2236
+    },
+    {
+      "epoch": 2.7840696950840074,
+      "grad_norm": 3.497357130050659,
+      "learning_rate": 4.358856088560886e-07,
+      "loss": 1.3452,
+      "step": 2237
+    },
+    {
+      "epoch": 2.7853142501555697,
+      "grad_norm": 3.4761931896209717,
+      "learning_rate": 4.3357933579335797e-07,
+      "loss": 1.3683,
+      "step": 2238
+    },
+    {
+      "epoch": 2.7865588052271315,
+      "grad_norm": 3.2924118041992188,
+      "learning_rate": 4.3127306273062734e-07,
+      "loss": 1.3321,
+      "step": 2239
+    },
+    {
+      "epoch": 2.7878033602986934,
+      "grad_norm": 3.4957733154296875,
+      "learning_rate": 4.289667896678967e-07,
+      "loss": 1.3366,
+      "step": 2240
+    },
+    {
+      "epoch": 2.7878033602986934,
+      "eval_loss": 1.6076867580413818,
+      "eval_runtime": 43.7189,
+      "eval_samples_per_second": 22.873,
+      "eval_steps_per_second": 0.961,
+      "step": 2240
+    },
+    {
+      "epoch": 2.789047915370255,
+      "grad_norm": 3.566307544708252,
+      "learning_rate": 4.266605166051661e-07,
+      "loss": 1.3584,
+      "step": 2241
+    },
+    {
+      "epoch": 2.790292470441817,
+      "grad_norm": 3.6012961864471436,
+      "learning_rate": 4.2435424354243545e-07,
+      "loss": 1.3666,
+      "step": 2242
+    },
+    {
+      "epoch": 2.791537025513379,
+      "grad_norm": 3.5159049034118652,
+      "learning_rate": 4.220479704797048e-07,
+      "loss": 1.3277,
+      "step": 2243
+    },
+    {
+      "epoch": 2.7927815805849407,
+      "grad_norm": 3.366673469543457,
+      "learning_rate": 4.197416974169742e-07,
+      "loss": 1.3685,
+      "step": 2244
+    },
+    {
+      "epoch": 2.7940261356565026,
+      "grad_norm": 3.439293384552002,
+      "learning_rate": 4.1743542435424356e-07,
+      "loss": 1.3611,
+      "step": 2245
+    },
+    {
+      "epoch": 2.7952706907280644,
+      "grad_norm": 3.3428847789764404,
+      "learning_rate": 4.1512915129151293e-07,
+      "loss": 1.3344,
+      "step": 2246
+    },
+    {
+      "epoch": 2.7965152457996267,
+      "grad_norm": 3.3846447467803955,
+      "learning_rate": 4.1282287822878235e-07,
+      "loss": 1.3298,
+      "step": 2247
+    },
+    {
+      "epoch": 2.7977598008711886,
+      "grad_norm": 3.4208669662475586,
+      "learning_rate": 4.1051660516605167e-07,
+      "loss": 1.3762,
+      "step": 2248
+    },
+    {
+      "epoch": 2.7990043559427504,
+      "grad_norm": 3.3218090534210205,
+      "learning_rate": 4.082103321033211e-07,
+      "loss": 1.2945,
+      "step": 2249
+    },
+    {
+      "epoch": 2.8002489110143123,
+      "grad_norm": 3.4152491092681885,
+      "learning_rate": 4.059040590405904e-07,
+      "loss": 1.3234,
+      "step": 2250
+    },
+    {
+      "epoch": 2.8002489110143123,
+      "eval_loss": 1.6081277132034302,
+      "eval_runtime": 43.957,
+      "eval_samples_per_second": 22.75,
+      "eval_steps_per_second": 0.955,
+      "step": 2250
+    },
+    {
+      "epoch": 2.8014934660858746,
+      "grad_norm": 3.408590316772461,
+      "learning_rate": 4.0359778597785983e-07,
+      "loss": 1.3457,
+      "step": 2251
+    },
+    {
+      "epoch": 2.8027380211574364,
+      "grad_norm": 3.53322172164917,
+      "learning_rate": 4.0129151291512915e-07,
+      "loss": 1.3905,
+      "step": 2252
+    },
+    {
+      "epoch": 2.8039825762289983,
+      "grad_norm": 3.3232524394989014,
+      "learning_rate": 3.9898523985239857e-07,
+      "loss": 1.298,
+      "step": 2253
+    },
+    {
+      "epoch": 2.80522713130056,
+      "grad_norm": 3.383641242980957,
+      "learning_rate": 3.966789667896679e-07,
+      "loss": 1.3425,
+      "step": 2254
+    },
+    {
+      "epoch": 2.806471686372122,
+      "grad_norm": 3.586348295211792,
+      "learning_rate": 3.943726937269373e-07,
+      "loss": 1.4114,
+      "step": 2255
+    },
+    {
+      "epoch": 2.807716241443684,
+      "grad_norm": 3.3169400691986084,
+      "learning_rate": 3.920664206642067e-07,
+      "loss": 1.3231,
+      "step": 2256
+    },
+    {
+      "epoch": 2.8089607965152457,
+      "grad_norm": 3.4631175994873047,
+      "learning_rate": 3.8976014760147605e-07,
+      "loss": 1.3547,
+      "step": 2257
+    },
+    {
+      "epoch": 2.8102053515868075,
+      "grad_norm": 3.609144687652588,
+      "learning_rate": 3.874538745387454e-07,
+      "loss": 1.3917,
+      "step": 2258
+    },
+    {
+      "epoch": 2.8114499066583694,
+      "grad_norm": 3.5235679149627686,
+      "learning_rate": 3.851476014760148e-07,
+      "loss": 1.358,
+      "step": 2259
+    },
+    {
+      "epoch": 2.8126944617299316,
+      "grad_norm": 3.3768770694732666,
+      "learning_rate": 3.8284132841328416e-07,
+      "loss": 1.39,
+      "step": 2260
+    },
+    {
+      "epoch": 2.8126944617299316,
+      "eval_loss": 1.6079868078231812,
+      "eval_runtime": 45.6946,
+      "eval_samples_per_second": 21.884,
+      "eval_steps_per_second": 0.919,
+      "step": 2260
+    },
+    {
+      "epoch": 2.8139390168014935,
+      "grad_norm": 3.3989570140838623,
+      "learning_rate": 3.8053505535055353e-07,
+      "loss": 1.3377,
+      "step": 2261
+    },
+    {
+      "epoch": 2.8151835718730553,
+      "grad_norm": 3.508310317993164,
+      "learning_rate": 3.782287822878229e-07,
+      "loss": 1.3407,
+      "step": 2262
+    },
+    {
+      "epoch": 2.816428126944617,
+      "grad_norm": 3.4873592853546143,
+      "learning_rate": 3.7592250922509227e-07,
+      "loss": 1.3646,
+      "step": 2263
+    },
+    {
+      "epoch": 2.8176726820161795,
+      "grad_norm": 3.4191977977752686,
+      "learning_rate": 3.7361623616236164e-07,
+      "loss": 1.311,
+      "step": 2264
+    },
+    {
+      "epoch": 2.8189172370877413,
+      "grad_norm": 3.394169330596924,
+      "learning_rate": 3.71309963099631e-07,
+      "loss": 1.333,
+      "step": 2265
+    },
+    {
+      "epoch": 2.820161792159303,
+      "grad_norm": 3.4663238525390625,
+      "learning_rate": 3.690036900369004e-07,
+      "loss": 1.3387,
+      "step": 2266
+    },
+    {
+      "epoch": 2.821406347230865,
+      "grad_norm": 3.4985597133636475,
+      "learning_rate": 3.666974169741698e-07,
+      "loss": 1.337,
+      "step": 2267
+    },
+    {
+      "epoch": 2.822650902302427,
+      "grad_norm": 3.3662006855010986,
+      "learning_rate": 3.643911439114391e-07,
+      "loss": 1.3339,
+      "step": 2268
+    },
+    {
+      "epoch": 2.8238954573739887,
+      "grad_norm": 3.384871482849121,
+      "learning_rate": 3.6208487084870854e-07,
+      "loss": 1.3116,
+      "step": 2269
+    },
+    {
+      "epoch": 2.8251400124455506,
+      "grad_norm": 3.3345062732696533,
+      "learning_rate": 3.5977859778597786e-07,
+      "loss": 1.2856,
+      "step": 2270
+    },
+    {
+      "epoch": 2.8251400124455506,
+      "eval_loss": 1.6083563566207886,
+      "eval_runtime": 46.5813,
+      "eval_samples_per_second": 21.468,
+      "eval_steps_per_second": 0.902,
+      "step": 2270
+    },
+    {
+      "epoch": 2.8263845675171124,
+      "grad_norm": 3.531426429748535,
+      "learning_rate": 3.574723247232473e-07,
+      "loss": 1.3625,
+      "step": 2271
+    },
+    {
+      "epoch": 2.8276291225886747,
+      "grad_norm": 3.508230447769165,
+      "learning_rate": 3.551660516605166e-07,
+      "loss": 1.3397,
+      "step": 2272
+    },
+    {
+      "epoch": 2.8288736776602366,
+      "grad_norm": 3.392129421234131,
+      "learning_rate": 3.52859778597786e-07,
+      "loss": 1.3,
+      "step": 2273
+    },
+    {
+      "epoch": 2.8301182327317984,
+      "grad_norm": 3.501406192779541,
+      "learning_rate": 3.5055350553505534e-07,
+      "loss": 1.3431,
+      "step": 2274
+    },
+    {
+      "epoch": 2.8313627878033603,
+      "grad_norm": 3.3904361724853516,
+      "learning_rate": 3.4824723247232476e-07,
+      "loss": 1.3497,
+      "step": 2275
+    },
+    {
+      "epoch": 2.832607342874922,
+      "grad_norm": 3.5180835723876953,
+      "learning_rate": 3.4594095940959413e-07,
+      "loss": 1.3638,
+      "step": 2276
+    },
+    {
+      "epoch": 2.8338518979464844,
+      "grad_norm": 3.3576693534851074,
+      "learning_rate": 3.436346863468635e-07,
+      "loss": 1.3294,
+      "step": 2277
+    },
+    {
+      "epoch": 2.8350964530180462,
+      "grad_norm": 3.4021854400634766,
+      "learning_rate": 3.413284132841329e-07,
+      "loss": 1.3155,
+      "step": 2278
+    },
+    {
+      "epoch": 2.836341008089608,
+      "grad_norm": 3.4220190048217773,
+      "learning_rate": 3.3902214022140224e-07,
+      "loss": 1.3419,
+      "step": 2279
+    },
+    {
+      "epoch": 2.83758556316117,
+      "grad_norm": 3.4883368015289307,
+      "learning_rate": 3.367158671586716e-07,
+      "loss": 1.3287,
+      "step": 2280
+    },
+    {
+      "epoch": 2.83758556316117,
+      "eval_loss": 1.6085011959075928,
+      "eval_runtime": 46.0344,
+      "eval_samples_per_second": 21.723,
+      "eval_steps_per_second": 0.912,
+      "step": 2280
+    },
+    {
+      "epoch": 2.838830118232732,
+      "grad_norm": 3.521838903427124,
+      "learning_rate": 3.34409594095941e-07,
+      "loss": 1.3278,
+      "step": 2281
+    },
+    {
+      "epoch": 2.8400746733042936,
+      "grad_norm": 3.420236587524414,
+      "learning_rate": 3.3210332103321035e-07,
+      "loss": 1.3021,
+      "step": 2282
+    },
+    {
+      "epoch": 2.8413192283758555,
+      "grad_norm": 3.2946665287017822,
+      "learning_rate": 3.297970479704797e-07,
+      "loss": 1.3223,
+      "step": 2283
+    },
+    {
+      "epoch": 2.8425637834474173,
+      "grad_norm": 3.435534715652466,
+      "learning_rate": 3.274907749077491e-07,
+      "loss": 1.358,
+      "step": 2284
+    },
+    {
+      "epoch": 2.8438083385189796,
+      "grad_norm": 3.48772931098938,
+      "learning_rate": 3.2518450184501846e-07,
+      "loss": 1.3427,
+      "step": 2285
+    },
+    {
+      "epoch": 2.8450528935905415,
+      "grad_norm": 3.41283917427063,
+      "learning_rate": 3.2287822878228783e-07,
+      "loss": 1.3057,
+      "step": 2286
+    },
+    {
+      "epoch": 2.8462974486621033,
+      "grad_norm": 3.2920539379119873,
+      "learning_rate": 3.2057195571955726e-07,
+      "loss": 1.3073,
+      "step": 2287
+    },
+    {
+      "epoch": 2.847542003733665,
+      "grad_norm": 3.385733127593994,
+      "learning_rate": 3.182656826568266e-07,
+      "loss": 1.3433,
+      "step": 2288
+    },
+    {
+      "epoch": 2.848786558805227,
+      "grad_norm": 3.486422538757324,
+      "learning_rate": 3.15959409594096e-07,
+      "loss": 1.373,
+      "step": 2289
+    },
+    {
+      "epoch": 2.8500311138767893,
+      "grad_norm": 3.4137535095214844,
+      "learning_rate": 3.136531365313653e-07,
+      "loss": 1.3352,
+      "step": 2290
+    },
+    {
+      "epoch": 2.8500311138767893,
+      "eval_loss": 1.607330322265625,
+      "eval_runtime": 45.4501,
+      "eval_samples_per_second": 22.002,
+      "eval_steps_per_second": 0.924,
+      "step": 2290
+    },
+    {
+      "epoch": 2.851275668948351,
+      "grad_norm": 3.474705219268799,
+      "learning_rate": 3.1134686346863474e-07,
+      "loss": 1.3808,
+      "step": 2291
+    },
+    {
+      "epoch": 2.852520224019913,
+      "grad_norm": 3.299457311630249,
+      "learning_rate": 3.090405904059041e-07,
+      "loss": 1.3343,
+      "step": 2292
+    },
+    {
+      "epoch": 2.853764779091475,
+      "grad_norm": 3.265301465988159,
+      "learning_rate": 3.067343173431735e-07,
+      "loss": 1.3106,
+      "step": 2293
+    },
+    {
+      "epoch": 2.8550093341630367,
+      "grad_norm": 3.213136911392212,
+      "learning_rate": 3.0442804428044285e-07,
+      "loss": 1.3102,
+      "step": 2294
+    },
+    {
+      "epoch": 2.8562538892345986,
+      "grad_norm": 3.365297794342041,
+      "learning_rate": 3.021217712177122e-07,
+      "loss": 1.323,
+      "step": 2295
+    },
+    {
+      "epoch": 2.8574984443061604,
+      "grad_norm": 3.331441879272461,
+      "learning_rate": 2.998154981549816e-07,
+      "loss": 1.2881,
+      "step": 2296
+    },
+    {
+      "epoch": 2.8587429993777222,
+      "grad_norm": 3.4592764377593994,
+      "learning_rate": 2.9750922509225096e-07,
+      "loss": 1.3134,
+      "step": 2297
+    },
+    {
+      "epoch": 2.8599875544492845,
+      "grad_norm": 3.5126025676727295,
+      "learning_rate": 2.9520295202952033e-07,
+      "loss": 1.3614,
+      "step": 2298
+    },
+    {
+      "epoch": 2.8612321095208464,
+      "grad_norm": 3.344198226928711,
+      "learning_rate": 2.928966789667897e-07,
+      "loss": 1.3389,
+      "step": 2299
+    },
+    {
+      "epoch": 2.8624766645924082,
+      "grad_norm": 3.404078722000122,
+      "learning_rate": 2.9059040590405907e-07,
+      "loss": 1.3592,
+      "step": 2300
+    },
+    {
+      "epoch": 2.8624766645924082,
+      "eval_loss": 1.608008623123169,
+      "eval_runtime": 47.7633,
+      "eval_samples_per_second": 20.937,
+      "eval_steps_per_second": 0.879,
+      "step": 2300
+    },
+    {
+      "epoch": 2.86372121966397,
+      "grad_norm": 3.492979049682617,
+      "learning_rate": 2.8828413284132844e-07,
+      "loss": 1.3857,
+      "step": 2301
+    },
+    {
+      "epoch": 2.864965774735532,
+      "grad_norm": 3.3371503353118896,
+      "learning_rate": 2.859778597785978e-07,
+      "loss": 1.3843,
+      "step": 2302
+    },
+    {
+      "epoch": 2.8662103298070942,
+      "grad_norm": 3.4544661045074463,
+      "learning_rate": 2.836715867158672e-07,
+      "loss": 1.3119,
+      "step": 2303
+    },
+    {
+      "epoch": 2.867454884878656,
+      "grad_norm": 3.363961696624756,
+      "learning_rate": 2.8136531365313655e-07,
+      "loss": 1.342,
+      "step": 2304
+    },
+    {
+      "epoch": 2.868699439950218,
+      "grad_norm": 3.3534724712371826,
+      "learning_rate": 2.790590405904059e-07,
+      "loss": 1.3361,
+      "step": 2305
+    },
+    {
+      "epoch": 2.8699439950217798,
+      "grad_norm": 3.345261573791504,
+      "learning_rate": 2.767527675276753e-07,
+      "loss": 1.3495,
+      "step": 2306
+    },
+    {
+      "epoch": 2.8711885500933416,
+      "grad_norm": 3.4351654052734375,
+      "learning_rate": 2.7444649446494466e-07,
+      "loss": 1.3353,
+      "step": 2307
+    },
+    {
+      "epoch": 2.8724331051649035,
+      "grad_norm": 3.5520448684692383,
+      "learning_rate": 2.72140221402214e-07,
+      "loss": 1.4051,
+      "step": 2308
+    },
+    {
+      "epoch": 2.8736776602364653,
+      "grad_norm": 3.389655113220215,
+      "learning_rate": 2.698339483394834e-07,
+      "loss": 1.3089,
+      "step": 2309
+    },
+    {
+      "epoch": 2.874922215308027,
+      "grad_norm": 3.451663017272949,
+      "learning_rate": 2.6752767527675277e-07,
+      "loss": 1.3256,
+      "step": 2310
+    },
+    {
+      "epoch": 2.874922215308027,
+      "eval_loss": 1.606674313545227,
+      "eval_runtime": 43.3577,
+      "eval_samples_per_second": 23.064,
+      "eval_steps_per_second": 0.969,
+      "step": 2310
+    },
+    {
+      "epoch": 2.8761667703795895,
+      "grad_norm": 3.479719877243042,
+      "learning_rate": 2.652214022140222e-07,
+      "loss": 1.336,
+      "step": 2311
+    },
+    {
+      "epoch": 2.8774113254511513,
+      "grad_norm": 3.341574192047119,
+      "learning_rate": 2.6291512915129156e-07,
+      "loss": 1.3037,
+      "step": 2312
+    },
+    {
+      "epoch": 2.878655880522713,
+      "grad_norm": 3.3325233459472656,
+      "learning_rate": 2.6060885608856093e-07,
+      "loss": 1.3238,
+      "step": 2313
+    },
+    {
+      "epoch": 2.879900435594275,
+      "grad_norm": 3.433773994445801,
+      "learning_rate": 2.583025830258303e-07,
+      "loss": 1.3414,
+      "step": 2314
+    },
+    {
+      "epoch": 2.881144990665837,
+      "grad_norm": 3.421121835708618,
+      "learning_rate": 2.5599630996309967e-07,
+      "loss": 1.3699,
+      "step": 2315
+    },
+    {
+      "epoch": 2.882389545737399,
+      "grad_norm": 3.5381970405578613,
+      "learning_rate": 2.5369003690036904e-07,
+      "loss": 1.421,
+      "step": 2316
+    },
+    {
+      "epoch": 2.883634100808961,
+      "grad_norm": 3.451042652130127,
+      "learning_rate": 2.513837638376384e-07,
+      "loss": 1.3518,
+      "step": 2317
+    },
+    {
+      "epoch": 2.884878655880523,
+      "grad_norm": 3.4976284503936768,
+      "learning_rate": 2.490774907749078e-07,
+      "loss": 1.3558,
+      "step": 2318
+    },
+    {
+      "epoch": 2.8861232109520847,
+      "grad_norm": 3.320169448852539,
+      "learning_rate": 2.4677121771217715e-07,
+      "loss": 1.2783,
+      "step": 2319
+    },
+    {
+      "epoch": 2.8873677660236465,
+      "grad_norm": 3.260979652404785,
+      "learning_rate": 2.444649446494465e-07,
+      "loss": 1.2932,
+      "step": 2320
+    },
+    {
+      "epoch": 2.8873677660236465,
+      "eval_loss": 1.6059826612472534,
+      "eval_runtime": 45.8271,
+      "eval_samples_per_second": 21.821,
+      "eval_steps_per_second": 0.916,
+      "step": 2320
+    },
+    {
+      "epoch": 2.8886123210952084,
+      "grad_norm": 3.347679376602173,
+      "learning_rate": 2.421586715867159e-07,
+      "loss": 1.3245,
+      "step": 2321
+    },
+    {
+      "epoch": 2.8898568761667702,
+      "grad_norm": 3.356933355331421,
+      "learning_rate": 2.3985239852398526e-07,
+      "loss": 1.3273,
+      "step": 2322
+    },
+    {
+      "epoch": 2.891101431238332,
+      "grad_norm": 3.406815528869629,
+      "learning_rate": 2.3754612546125463e-07,
+      "loss": 1.3657,
+      "step": 2323
+    },
+    {
+      "epoch": 2.8923459863098944,
+      "grad_norm": 3.2950358390808105,
+      "learning_rate": 2.35239852398524e-07,
+      "loss": 1.3093,
+      "step": 2324
+    },
+    {
+      "epoch": 2.893590541381456,
+      "grad_norm": 3.5412557125091553,
+      "learning_rate": 2.3293357933579337e-07,
+      "loss": 1.3832,
+      "step": 2325
+    },
+    {
+      "epoch": 2.894835096453018,
+      "grad_norm": 3.5626139640808105,
+      "learning_rate": 2.3062730627306274e-07,
+      "loss": 1.4213,
+      "step": 2326
+    },
+    {
+      "epoch": 2.89607965152458,
+      "grad_norm": 3.365360975265503,
+      "learning_rate": 2.283210332103321e-07,
+      "loss": 1.3407,
+      "step": 2327
+    },
+    {
+      "epoch": 2.8973242065961418,
+      "grad_norm": 3.430964946746826,
+      "learning_rate": 2.2601476014760148e-07,
+      "loss": 1.3769,
+      "step": 2328
+    },
+    {
+      "epoch": 2.898568761667704,
+      "grad_norm": 3.5526304244995117,
+      "learning_rate": 2.2370848708487085e-07,
+      "loss": 1.3634,
+      "step": 2329
+    },
+    {
+      "epoch": 2.899813316739266,
+      "grad_norm": 3.5068812370300293,
+      "learning_rate": 2.2140221402214022e-07,
+      "loss": 1.3586,
+      "step": 2330
+    },
+    {
+      "epoch": 2.899813316739266,
+      "eval_loss": 1.6059414148330688,
+      "eval_runtime": 47.2048,
+      "eval_samples_per_second": 21.184,
+      "eval_steps_per_second": 0.89,
+      "step": 2330
+    },
+    {
+      "epoch": 2.9010578718108277,
+      "grad_norm": 3.408522367477417,
+      "learning_rate": 2.1909594095940962e-07,
+      "loss": 1.3321,
+      "step": 2331
+    },
+    {
+      "epoch": 2.9023024268823896,
+      "grad_norm": 3.361820697784424,
+      "learning_rate": 2.1678966789667899e-07,
+      "loss": 1.3175,
+      "step": 2332
+    },
+    {
+      "epoch": 2.9035469819539514,
+      "grad_norm": 3.428621292114258,
+      "learning_rate": 2.1448339483394835e-07,
+      "loss": 1.3369,
+      "step": 2333
+    },
+    {
+      "epoch": 2.9047915370255133,
+      "grad_norm": 3.498232126235962,
+      "learning_rate": 2.1217712177121772e-07,
+      "loss": 1.3163,
+      "step": 2334
+    },
+    {
+      "epoch": 2.906036092097075,
+      "grad_norm": 3.2694506645202637,
+      "learning_rate": 2.098708487084871e-07,
+      "loss": 1.305,
+      "step": 2335
+    },
+    {
+      "epoch": 2.907280647168637,
+      "grad_norm": 3.5026626586914062,
+      "learning_rate": 2.0756457564575646e-07,
+      "loss": 1.3421,
+      "step": 2336
+    },
+    {
+      "epoch": 2.9085252022401993,
+      "grad_norm": 3.4477906227111816,
+      "learning_rate": 2.0525830258302583e-07,
+      "loss": 1.3832,
+      "step": 2337
+    },
+    {
+      "epoch": 2.909769757311761,
+      "grad_norm": 3.451176166534424,
+      "learning_rate": 2.029520295202952e-07,
+      "loss": 1.3529,
+      "step": 2338
+    },
+    {
+      "epoch": 2.911014312383323,
+      "grad_norm": 3.452903985977173,
+      "learning_rate": 2.0064575645756457e-07,
+      "loss": 1.2761,
+      "step": 2339
+    },
+    {
+      "epoch": 2.912258867454885,
+      "grad_norm": 3.414515733718872,
+      "learning_rate": 1.9833948339483394e-07,
+      "loss": 1.3349,
+      "step": 2340
+    },
+    {
+      "epoch": 2.912258867454885,
+      "eval_loss": 1.6055874824523926,
+      "eval_runtime": 48.3953,
+      "eval_samples_per_second": 20.663,
+      "eval_steps_per_second": 0.868,
+      "step": 2340
+    },
+    {
+      "epoch": 2.9135034225264467,
+      "grad_norm": 3.342360019683838,
+      "learning_rate": 1.9603321033210334e-07,
+      "loss": 1.334,
+      "step": 2341
+    },
+    {
+      "epoch": 2.914747977598009,
+      "grad_norm": 3.5278728008270264,
+      "learning_rate": 1.937269372693727e-07,
+      "loss": 1.3787,
+      "step": 2342
+    },
+    {
+      "epoch": 2.915992532669571,
+      "grad_norm": 3.310112476348877,
+      "learning_rate": 1.9142066420664208e-07,
+      "loss": 1.2401,
+      "step": 2343
+    },
+    {
+      "epoch": 2.9172370877411327,
+      "grad_norm": 3.395261764526367,
+      "learning_rate": 1.8911439114391145e-07,
+      "loss": 1.3416,
+      "step": 2344
+    },
+    {
+      "epoch": 2.9184816428126945,
+      "grad_norm": 3.380107879638672,
+      "learning_rate": 1.8680811808118082e-07,
+      "loss": 1.3134,
+      "step": 2345
+    },
+    {
+      "epoch": 2.9197261978842564,
+      "grad_norm": 3.4059510231018066,
+      "learning_rate": 1.845018450184502e-07,
+      "loss": 1.3452,
+      "step": 2346
+    },
+    {
+      "epoch": 2.920970752955818,
+      "grad_norm": 3.382456064224243,
+      "learning_rate": 1.8219557195571956e-07,
+      "loss": 1.3497,
+      "step": 2347
+    },
+    {
+      "epoch": 2.92221530802738,
+      "grad_norm": 3.3745336532592773,
+      "learning_rate": 1.7988929889298893e-07,
+      "loss": 1.2968,
+      "step": 2348
+    },
+    {
+      "epoch": 2.923459863098942,
+      "grad_norm": 3.4328863620758057,
+      "learning_rate": 1.775830258302583e-07,
+      "loss": 1.3086,
+      "step": 2349
+    },
+    {
+      "epoch": 2.924704418170504,
+      "grad_norm": 3.4179446697235107,
+      "learning_rate": 1.7527675276752767e-07,
+      "loss": 1.3552,
+      "step": 2350
+    },
+    {
+      "epoch": 2.924704418170504,
+      "eval_loss": 1.6056684255599976,
+      "eval_runtime": 50.027,
+      "eval_samples_per_second": 19.989,
+      "eval_steps_per_second": 0.84,
+      "step": 2350
+    },
+    {
+      "epoch": 2.925948973242066,
+      "grad_norm": 3.281198263168335,
+      "learning_rate": 1.7297047970479707e-07,
+      "loss": 1.335,
+      "step": 2351
+    },
+    {
+      "epoch": 2.927193528313628,
+      "grad_norm": 3.467276096343994,
+      "learning_rate": 1.7066420664206644e-07,
+      "loss": 1.3371,
+      "step": 2352
+    },
+    {
+      "epoch": 2.9284380833851897,
+      "grad_norm": 3.3215816020965576,
+      "learning_rate": 1.683579335793358e-07,
+      "loss": 1.3769,
+      "step": 2353
+    },
+    {
+      "epoch": 2.9296826384567516,
+      "grad_norm": 3.34025502204895,
+      "learning_rate": 1.6605166051660518e-07,
+      "loss": 1.3066,
+      "step": 2354
+    },
+    {
+      "epoch": 2.930927193528314,
+      "grad_norm": 3.4006409645080566,
+      "learning_rate": 1.6374538745387455e-07,
+      "loss": 1.3429,
+      "step": 2355
+    },
+    {
+      "epoch": 2.9321717485998757,
+      "grad_norm": 3.6019599437713623,
+      "learning_rate": 1.6143911439114392e-07,
+      "loss": 1.3744,
+      "step": 2356
+    },
+    {
+      "epoch": 2.9334163036714376,
+      "grad_norm": 3.4688234329223633,
+      "learning_rate": 1.591328413284133e-07,
+      "loss": 1.365,
+      "step": 2357
+    },
+    {
+      "epoch": 2.9346608587429994,
+      "grad_norm": 3.3212244510650635,
+      "learning_rate": 1.5682656826568266e-07,
+      "loss": 1.3062,
+      "step": 2358
+    },
+    {
+      "epoch": 2.9359054138145613,
+      "grad_norm": 3.4752397537231445,
+      "learning_rate": 1.5452029520295205e-07,
+      "loss": 1.3546,
+      "step": 2359
+    },
+    {
+      "epoch": 2.937149968886123,
+      "grad_norm": 3.4061782360076904,
+      "learning_rate": 1.5221402214022142e-07,
+      "loss": 1.3463,
+      "step": 2360
+    },
+    {
+      "epoch": 2.937149968886123,
+      "eval_loss": 1.6056970357894897,
+      "eval_runtime": 53.757,
+      "eval_samples_per_second": 18.602,
+      "eval_steps_per_second": 0.781,
+      "step": 2360
+    },
+    {
+      "epoch": 2.938394523957685,
+      "grad_norm": 3.51235294342041,
+      "learning_rate": 1.499077490774908e-07,
+      "loss": 1.3574,
+      "step": 2361
+    },
+    {
+      "epoch": 2.939639079029247,
+      "grad_norm": 3.5272417068481445,
+      "learning_rate": 1.4760147601476016e-07,
+      "loss": 1.3447,
+      "step": 2362
+    },
+    {
+      "epoch": 2.940883634100809,
+      "grad_norm": 3.4143166542053223,
+      "learning_rate": 1.4529520295202953e-07,
+      "loss": 1.3563,
+      "step": 2363
+    },
+    {
+      "epoch": 2.942128189172371,
+      "grad_norm": 3.307703733444214,
+      "learning_rate": 1.429889298892989e-07,
+      "loss": 1.3035,
+      "step": 2364
+    },
+    {
+      "epoch": 2.943372744243933,
+      "grad_norm": 3.4650306701660156,
+      "learning_rate": 1.4068265682656827e-07,
+      "loss": 1.3512,
+      "step": 2365
+    },
+    {
+      "epoch": 2.9446172993154947,
+      "grad_norm": 3.3445703983306885,
+      "learning_rate": 1.3837638376383764e-07,
+      "loss": 1.298,
+      "step": 2366
+    },
+    {
+      "epoch": 2.9458618543870565,
+      "grad_norm": 3.379523277282715,
+      "learning_rate": 1.36070110701107e-07,
+      "loss": 1.3191,
+      "step": 2367
+    },
+    {
+      "epoch": 2.947106409458619,
+      "grad_norm": 3.2469892501831055,
+      "learning_rate": 1.3376383763837638e-07,
+      "loss": 1.3134,
+      "step": 2368
+    },
+    {
+      "epoch": 2.9483509645301806,
+      "grad_norm": 3.354652166366577,
+      "learning_rate": 1.3145756457564578e-07,
+      "loss": 1.2706,
+      "step": 2369
+    },
+    {
+      "epoch": 2.9495955196017425,
+      "grad_norm": 3.519868850708008,
+      "learning_rate": 1.2915129151291515e-07,
+      "loss": 1.3341,
+      "step": 2370
+    },
+    {
+      "epoch": 2.9495955196017425,
+      "eval_loss": 1.6041405200958252,
+      "eval_runtime": 53.7598,
+      "eval_samples_per_second": 18.601,
+      "eval_steps_per_second": 0.781,
+      "step": 2370
+    },
+    {
+      "epoch": 2.9508400746733043,
+      "grad_norm": 3.3218917846679688,
+      "learning_rate": 1.2684501845018452e-07,
+      "loss": 1.3012,
+      "step": 2371
+    },
+    {
+      "epoch": 2.952084629744866,
+      "grad_norm": 3.3390657901763916,
+      "learning_rate": 1.245387453874539e-07,
+      "loss": 1.2808,
+      "step": 2372
+    },
+    {
+      "epoch": 2.953329184816428,
+      "grad_norm": 3.473332405090332,
+      "learning_rate": 1.2223247232472326e-07,
+      "loss": 1.4012,
+      "step": 2373
+    },
+    {
+      "epoch": 2.95457373988799,
+      "grad_norm": 3.3906960487365723,
+      "learning_rate": 1.1992619926199263e-07,
+      "loss": 1.3382,
+      "step": 2374
+    },
+    {
+      "epoch": 2.9558182949595517,
+      "grad_norm": 3.3417513370513916,
+      "learning_rate": 1.17619926199262e-07,
+      "loss": 1.3105,
+      "step": 2375
+    },
+    {
+      "epoch": 2.957062850031114,
+      "grad_norm": 3.3096163272857666,
+      "learning_rate": 1.1531365313653137e-07,
+      "loss": 1.2863,
+      "step": 2376
+    },
+    {
+      "epoch": 2.958307405102676,
+      "grad_norm": 3.401026964187622,
+      "learning_rate": 1.1300738007380074e-07,
+      "loss": 1.3164,
+      "step": 2377
+    },
+    {
+      "epoch": 2.9595519601742377,
+      "grad_norm": 3.3502590656280518,
+      "learning_rate": 1.1070110701107011e-07,
+      "loss": 1.3336,
+      "step": 2378
+    },
+    {
+      "epoch": 2.9607965152457996,
+      "grad_norm": 3.3403470516204834,
+      "learning_rate": 1.0839483394833949e-07,
+      "loss": 1.3034,
+      "step": 2379
+    },
+    {
+      "epoch": 2.9620410703173614,
+      "grad_norm": 3.4176461696624756,
+      "learning_rate": 1.0608856088560886e-07,
+      "loss": 1.3249,
+      "step": 2380
+    },
+    {
+      "epoch": 2.9620410703173614,
+      "eval_loss": 1.6039983034133911,
+      "eval_runtime": 53.2205,
+      "eval_samples_per_second": 18.79,
+      "eval_steps_per_second": 0.789,
+      "step": 2380
+    },
+    {
+      "epoch": 2.9632856253889237,
+      "grad_norm": 3.5640950202941895,
+      "learning_rate": 1.0378228782287823e-07,
+      "loss": 1.417,
+      "step": 2381
+    },
+    {
+      "epoch": 2.9645301804604856,
+      "grad_norm": 3.3958089351654053,
+      "learning_rate": 1.014760147601476e-07,
+      "loss": 1.3449,
+      "step": 2382
+    },
+    {
+      "epoch": 2.9657747355320474,
+      "grad_norm": 3.3931593894958496,
+      "learning_rate": 9.916974169741697e-08,
+      "loss": 1.345,
+      "step": 2383
+    },
+    {
+      "epoch": 2.9670192906036092,
+      "grad_norm": 3.2931392192840576,
+      "learning_rate": 9.686346863468636e-08,
+      "loss": 1.2974,
+      "step": 2384
+    },
+    {
+      "epoch": 2.968263845675171,
+      "grad_norm": 3.3938586711883545,
+      "learning_rate": 9.455719557195573e-08,
+      "loss": 1.3238,
+      "step": 2385
+    },
+    {
+      "epoch": 2.969508400746733,
+      "grad_norm": 3.4670567512512207,
+      "learning_rate": 9.22509225092251e-08,
+      "loss": 1.3489,
+      "step": 2386
+    },
+    {
+      "epoch": 2.970752955818295,
+      "grad_norm": 3.394413471221924,
+      "learning_rate": 8.994464944649447e-08,
+      "loss": 1.3319,
+      "step": 2387
+    },
+    {
+      "epoch": 2.9719975108898566,
+      "grad_norm": 3.5549511909484863,
+      "learning_rate": 8.763837638376384e-08,
+      "loss": 1.3342,
+      "step": 2388
+    },
+    {
+      "epoch": 2.973242065961419,
+      "grad_norm": 3.501979351043701,
+      "learning_rate": 8.533210332103322e-08,
+      "loss": 1.379,
+      "step": 2389
+    },
+    {
+      "epoch": 2.974486621032981,
+      "grad_norm": 3.274027109146118,
+      "learning_rate": 8.302583025830259e-08,
+      "loss": 1.3175,
+      "step": 2390
+    },
+    {
+      "epoch": 2.974486621032981,
+      "eval_loss": 1.6037445068359375,
+      "eval_runtime": 51.3867,
+      "eval_samples_per_second": 19.46,
+      "eval_steps_per_second": 0.817,
+      "step": 2390
+    },
+    {
+      "epoch": 2.9757311761045426,
+      "grad_norm": 3.5105645656585693,
+      "learning_rate": 8.071955719557196e-08,
+      "loss": 1.2838,
+      "step": 2391
+    },
+    {
+      "epoch": 2.9769757311761045,
+      "grad_norm": 3.4089620113372803,
+      "learning_rate": 7.841328413284133e-08,
+      "loss": 1.3109,
+      "step": 2392
+    },
+    {
+      "epoch": 2.9782202862476663,
+      "grad_norm": 3.516371488571167,
+      "learning_rate": 7.610701107011071e-08,
+      "loss": 1.3934,
+      "step": 2393
+    },
+    {
+      "epoch": 2.9794648413192286,
+      "grad_norm": 3.450157880783081,
+      "learning_rate": 7.380073800738008e-08,
+      "loss": 1.4141,
+      "step": 2394
+    },
+    {
+      "epoch": 2.9807093963907905,
+      "grad_norm": 3.4195621013641357,
+      "learning_rate": 7.149446494464945e-08,
+      "loss": 1.3804,
+      "step": 2395
+    },
+    {
+      "epoch": 2.9819539514623523,
+      "grad_norm": 3.465637445449829,
+      "learning_rate": 6.918819188191882e-08,
+      "loss": 1.4012,
+      "step": 2396
+    },
+    {
+      "epoch": 2.983198506533914,
+      "grad_norm": 3.4396843910217285,
+      "learning_rate": 6.688191881918819e-08,
+      "loss": 1.3612,
+      "step": 2397
+    },
+    {
+      "epoch": 2.984443061605476,
+      "grad_norm": 3.3771772384643555,
+      "learning_rate": 6.457564575645757e-08,
+      "loss": 1.3409,
+      "step": 2398
+    },
+    {
+      "epoch": 2.985687616677038,
+      "grad_norm": 3.2686965465545654,
+      "learning_rate": 6.226937269372694e-08,
+      "loss": 1.3168,
+      "step": 2399
+    },
+    {
+      "epoch": 2.9869321717485997,
+      "grad_norm": 3.4644386768341064,
+      "learning_rate": 5.996309963099631e-08,
+      "loss": 1.329,
+      "step": 2400
+    },
+    {
+      "epoch": 2.9869321717485997,
+      "eval_loss": 1.603417992591858,
+      "eval_runtime": 48.9725,
+      "eval_samples_per_second": 20.42,
+      "eval_steps_per_second": 0.858,
+      "step": 2400
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2409,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.481542582468608e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-2400/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/latest b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/latest
new file mode 100644
index 0000000000000000000000000000000000000000..6761b575fffac7f1984044dcb6446b3a51da04c8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/latest
@@ -0,0 +1 @@
+global_step300
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9015a2b291768ac1cbe0fb417354be7e82849ff8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c47a2aec04adfb2b774bed1d02a5453693feb66e4636bdf7cd908de748732842
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a17e6802aa777eb35577f53d24a38a0381fecbb7
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e375d2646839f0fc8836323cd9f6c6fec1f1316a3f21b7366755c68ab3c469c
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..8aab09805997622aec2edae7b565ed82010cfd53
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6802d0e782c931221303c74054cdedbab70d850d6310dc553b393a4f3f848807
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..744393bca39d9362bfbfc53004854f260d4632c3
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:335b00ec8932bdb6d648bd88e1e183f3921e9ff37f9da26a25679f12a101c4ed
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..f28ecaeab53ae07feed29ccf8624d2b0a8344df9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19fb2e1e3cdd6f7433d89fd6d62c82042599dd4984f342efe7fec6e159e6a8f6
+size 17210734
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..8991b49e9c2a43fc527dab9e09ad8171f0cc5943
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/tokenizer_config.json
@@ -0,0 +1,2086 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "🅂",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128258": {
+      "content": "🄿",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..47ee38a9cf9ee5f73a3d72fd32a1586b09383a18
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/trainer_state.json
@@ -0,0 +1,2373 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.37336652146857496,
+  "eval_steps": 10,
+  "global_step": 300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8368,
+      "step": 1
+    },
+    {
+      "epoch": 0.002489110143123833,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.86,
+      "step": 2
+    },
+    {
+      "epoch": 0.00373366521468575,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8265,
+      "step": 3
+    },
+    {
+      "epoch": 0.004978220286247666,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8271,
+      "step": 4
+    },
+    {
+      "epoch": 0.006222775357809583,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8768,
+      "step": 5
+    },
+    {
+      "epoch": 0.0074673304293715,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8402,
+      "step": 6
+    },
+    {
+      "epoch": 0.008711885500933417,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.785,
+      "step": 7
+    },
+    {
+      "epoch": 0.009956440572495333,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8466,
+      "step": 8
+    },
+    {
+      "epoch": 0.01120099564405725,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.7673,
+      "step": 9
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.012445550715619166,
+      "eval_loss": 2.8535053730010986,
+      "eval_runtime": 42.446,
+      "eval_samples_per_second": 23.559,
+      "eval_steps_per_second": 0.989,
+      "step": 10
+    },
+    {
+      "epoch": 0.013690105787181083,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8592,
+      "step": 11
+    },
+    {
+      "epoch": 0.014934660858743,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.804,
+      "step": 12
+    },
+    {
+      "epoch": 0.016179215930304917,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8076,
+      "step": 13
+    },
+    {
+      "epoch": 0.017423771001866834,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 2.8219,
+      "step": 14
+    },
+    {
+      "epoch": 0.018668326073428748,
+      "grad_norm": 5.105710506439209,
+      "learning_rate": 2.0746887966804982e-08,
+      "loss": 2.7995,
+      "step": 15
+    },
+    {
+      "epoch": 0.019912881144990666,
+      "grad_norm": 5.156588077545166,
+      "learning_rate": 4.1493775933609963e-08,
+      "loss": 2.8577,
+      "step": 16
+    },
+    {
+      "epoch": 0.021157436216552583,
+      "grad_norm": 5.709373950958252,
+      "learning_rate": 6.224066390041494e-08,
+      "loss": 2.8433,
+      "step": 17
+    },
+    {
+      "epoch": 0.0224019912881145,
+      "grad_norm": 5.64658784866333,
+      "learning_rate": 8.298755186721993e-08,
+      "loss": 2.8428,
+      "step": 18
+    },
+    {
+      "epoch": 0.023646546359676415,
+      "grad_norm": 5.541937351226807,
+      "learning_rate": 1.037344398340249e-07,
+      "loss": 2.7785,
+      "step": 19
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "grad_norm": 5.233374118804932,
+      "learning_rate": 1.2448132780082988e-07,
+      "loss": 2.8599,
+      "step": 20
+    },
+    {
+      "epoch": 0.024891101431238332,
+      "eval_loss": 2.8521182537078857,
+      "eval_runtime": 43.5664,
+      "eval_samples_per_second": 22.953,
+      "eval_steps_per_second": 0.964,
+      "step": 20
+    },
+    {
+      "epoch": 0.02613565650280025,
+      "grad_norm": 5.407598972320557,
+      "learning_rate": 1.4522821576763488e-07,
+      "loss": 2.8081,
+      "step": 21
+    },
+    {
+      "epoch": 0.027380211574362167,
+      "grad_norm": 5.0483317375183105,
+      "learning_rate": 1.6597510373443985e-07,
+      "loss": 2.8489,
+      "step": 22
+    },
+    {
+      "epoch": 0.02862476664592408,
+      "grad_norm": 4.985588550567627,
+      "learning_rate": 1.8672199170124483e-07,
+      "loss": 2.8375,
+      "step": 23
+    },
+    {
+      "epoch": 0.029869321717486,
+      "grad_norm": 5.201700687408447,
+      "learning_rate": 2.074688796680498e-07,
+      "loss": 2.8355,
+      "step": 24
+    },
+    {
+      "epoch": 0.031113876789047916,
+      "grad_norm": 5.009235382080078,
+      "learning_rate": 2.2821576763485478e-07,
+      "loss": 2.7774,
+      "step": 25
+    },
+    {
+      "epoch": 0.03235843186060983,
+      "grad_norm": 4.509210109710693,
+      "learning_rate": 2.4896265560165975e-07,
+      "loss": 2.7992,
+      "step": 26
+    },
+    {
+      "epoch": 0.03360298693217175,
+      "grad_norm": 4.529262065887451,
+      "learning_rate": 2.6970954356846476e-07,
+      "loss": 2.836,
+      "step": 27
+    },
+    {
+      "epoch": 0.03484754200373367,
+      "grad_norm": 4.526787281036377,
+      "learning_rate": 2.9045643153526976e-07,
+      "loss": 2.8128,
+      "step": 28
+    },
+    {
+      "epoch": 0.03609209707529558,
+      "grad_norm": 4.016609191894531,
+      "learning_rate": 3.112033195020747e-07,
+      "loss": 2.7947,
+      "step": 29
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "grad_norm": 3.4784562587738037,
+      "learning_rate": 3.319502074688797e-07,
+      "loss": 2.7854,
+      "step": 30
+    },
+    {
+      "epoch": 0.037336652146857496,
+      "eval_loss": 2.8020100593566895,
+      "eval_runtime": 44.8464,
+      "eval_samples_per_second": 22.298,
+      "eval_steps_per_second": 0.937,
+      "step": 30
+    },
+    {
+      "epoch": 0.038581207218419414,
+      "grad_norm": 3.4887945652008057,
+      "learning_rate": 3.5269709543568466e-07,
+      "loss": 2.7908,
+      "step": 31
+    },
+    {
+      "epoch": 0.03982576228998133,
+      "grad_norm": 3.4533374309539795,
+      "learning_rate": 3.7344398340248966e-07,
+      "loss": 2.7358,
+      "step": 32
+    },
+    {
+      "epoch": 0.04107031736154325,
+      "grad_norm": 3.270954132080078,
+      "learning_rate": 3.941908713692946e-07,
+      "loss": 2.8013,
+      "step": 33
+    },
+    {
+      "epoch": 0.042314872433105166,
+      "grad_norm": 3.61322283744812,
+      "learning_rate": 4.149377593360996e-07,
+      "loss": 2.7521,
+      "step": 34
+    },
+    {
+      "epoch": 0.043559427504667084,
+      "grad_norm": 3.158719062805176,
+      "learning_rate": 4.3568464730290456e-07,
+      "loss": 2.7573,
+      "step": 35
+    },
+    {
+      "epoch": 0.044803982576229,
+      "grad_norm": 2.888746500015259,
+      "learning_rate": 4.5643153526970956e-07,
+      "loss": 2.7795,
+      "step": 36
+    },
+    {
+      "epoch": 0.04604853764779091,
+      "grad_norm": 3.022629499435425,
+      "learning_rate": 4.771784232365145e-07,
+      "loss": 2.7891,
+      "step": 37
+    },
+    {
+      "epoch": 0.04729309271935283,
+      "grad_norm": 3.038097381591797,
+      "learning_rate": 4.979253112033195e-07,
+      "loss": 2.7376,
+      "step": 38
+    },
+    {
+      "epoch": 0.04853764779091475,
+      "grad_norm": 2.9992239475250244,
+      "learning_rate": 5.186721991701245e-07,
+      "loss": 2.7279,
+      "step": 39
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "grad_norm": 2.908842086791992,
+      "learning_rate": 5.394190871369295e-07,
+      "loss": 2.6582,
+      "step": 40
+    },
+    {
+      "epoch": 0.049782202862476664,
+      "eval_loss": 2.725968837738037,
+      "eval_runtime": 44.9498,
+      "eval_samples_per_second": 22.247,
+      "eval_steps_per_second": 0.934,
+      "step": 40
+    },
+    {
+      "epoch": 0.05102675793403858,
+      "grad_norm": 2.8025174140930176,
+      "learning_rate": 5.601659751037345e-07,
+      "loss": 2.6809,
+      "step": 41
+    },
+    {
+      "epoch": 0.0522713130056005,
+      "grad_norm": 2.7555699348449707,
+      "learning_rate": 5.809128630705395e-07,
+      "loss": 2.6954,
+      "step": 42
+    },
+    {
+      "epoch": 0.053515868077162417,
+      "grad_norm": 2.4732933044433594,
+      "learning_rate": 6.016597510373444e-07,
+      "loss": 2.7307,
+      "step": 43
+    },
+    {
+      "epoch": 0.054760423148724334,
+      "grad_norm": 2.451366424560547,
+      "learning_rate": 6.224066390041494e-07,
+      "loss": 2.7064,
+      "step": 44
+    },
+    {
+      "epoch": 0.056004978220286245,
+      "grad_norm": 2.5768678188323975,
+      "learning_rate": 6.431535269709543e-07,
+      "loss": 2.6466,
+      "step": 45
+    },
+    {
+      "epoch": 0.05724953329184816,
+      "grad_norm": 2.579332113265991,
+      "learning_rate": 6.639004149377594e-07,
+      "loss": 2.6127,
+      "step": 46
+    },
+    {
+      "epoch": 0.05849408836341008,
+      "grad_norm": 2.231207847595215,
+      "learning_rate": 6.846473029045644e-07,
+      "loss": 2.6429,
+      "step": 47
+    },
+    {
+      "epoch": 0.059738643434972,
+      "grad_norm": 2.134375810623169,
+      "learning_rate": 7.053941908713693e-07,
+      "loss": 2.6423,
+      "step": 48
+    },
+    {
+      "epoch": 0.060983198506533914,
+      "grad_norm": 2.0485494136810303,
+      "learning_rate": 7.261410788381744e-07,
+      "loss": 2.581,
+      "step": 49
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "grad_norm": 2.02909517288208,
+      "learning_rate": 7.468879668049793e-07,
+      "loss": 2.6357,
+      "step": 50
+    },
+    {
+      "epoch": 0.06222775357809583,
+      "eval_loss": 2.632822036743164,
+      "eval_runtime": 47.0104,
+      "eval_samples_per_second": 21.272,
+      "eval_steps_per_second": 0.893,
+      "step": 50
+    },
+    {
+      "epoch": 0.06347230864965775,
+      "grad_norm": 1.892500877380371,
+      "learning_rate": 7.676348547717843e-07,
+      "loss": 2.5945,
+      "step": 51
+    },
+    {
+      "epoch": 0.06471686372121967,
+      "grad_norm": 2.107921600341797,
+      "learning_rate": 7.883817427385892e-07,
+      "loss": 2.5867,
+      "step": 52
+    },
+    {
+      "epoch": 0.06596141879278158,
+      "grad_norm": 1.8287527561187744,
+      "learning_rate": 8.091286307053943e-07,
+      "loss": 2.5862,
+      "step": 53
+    },
+    {
+      "epoch": 0.0672059738643435,
+      "grad_norm": 1.7342065572738647,
+      "learning_rate": 8.298755186721992e-07,
+      "loss": 2.5735,
+      "step": 54
+    },
+    {
+      "epoch": 0.06845052893590542,
+      "grad_norm": 1.7905038595199585,
+      "learning_rate": 8.506224066390042e-07,
+      "loss": 2.572,
+      "step": 55
+    },
+    {
+      "epoch": 0.06969508400746734,
+      "grad_norm": 1.9715144634246826,
+      "learning_rate": 8.713692946058091e-07,
+      "loss": 2.5797,
+      "step": 56
+    },
+    {
+      "epoch": 0.07093963907902924,
+      "grad_norm": 1.788966417312622,
+      "learning_rate": 8.921161825726142e-07,
+      "loss": 2.5567,
+      "step": 57
+    },
+    {
+      "epoch": 0.07218419415059116,
+      "grad_norm": 1.8978915214538574,
+      "learning_rate": 9.128630705394191e-07,
+      "loss": 2.5776,
+      "step": 58
+    },
+    {
+      "epoch": 0.07342874922215308,
+      "grad_norm": 1.6401689052581787,
+      "learning_rate": 9.336099585062241e-07,
+      "loss": 2.5662,
+      "step": 59
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "grad_norm": 1.8200798034667969,
+      "learning_rate": 9.54356846473029e-07,
+      "loss": 2.5001,
+      "step": 60
+    },
+    {
+      "epoch": 0.07467330429371499,
+      "eval_loss": 2.569110870361328,
+      "eval_runtime": 47.3292,
+      "eval_samples_per_second": 21.129,
+      "eval_steps_per_second": 0.887,
+      "step": 60
+    },
+    {
+      "epoch": 0.07591785936527691,
+      "grad_norm": 1.5468674898147583,
+      "learning_rate": 9.751037344398341e-07,
+      "loss": 2.5308,
+      "step": 61
+    },
+    {
+      "epoch": 0.07716241443683883,
+      "grad_norm": 1.723684310913086,
+      "learning_rate": 9.95850622406639e-07,
+      "loss": 2.4907,
+      "step": 62
+    },
+    {
+      "epoch": 0.07840696950840075,
+      "grad_norm": 1.6346293687820435,
+      "learning_rate": 1.0165975103734441e-06,
+      "loss": 2.514,
+      "step": 63
+    },
+    {
+      "epoch": 0.07965152457996266,
+      "grad_norm": 1.8902088403701782,
+      "learning_rate": 1.037344398340249e-06,
+      "loss": 2.4859,
+      "step": 64
+    },
+    {
+      "epoch": 0.08089607965152458,
+      "grad_norm": 2.092611312866211,
+      "learning_rate": 1.058091286307054e-06,
+      "loss": 2.4888,
+      "step": 65
+    },
+    {
+      "epoch": 0.0821406347230865,
+      "grad_norm": 1.758482575416565,
+      "learning_rate": 1.078838174273859e-06,
+      "loss": 2.4537,
+      "step": 66
+    },
+    {
+      "epoch": 0.08338518979464841,
+      "grad_norm": 1.8941714763641357,
+      "learning_rate": 1.099585062240664e-06,
+      "loss": 2.5416,
+      "step": 67
+    },
+    {
+      "epoch": 0.08462974486621033,
+      "grad_norm": 1.7030234336853027,
+      "learning_rate": 1.120331950207469e-06,
+      "loss": 2.4791,
+      "step": 68
+    },
+    {
+      "epoch": 0.08587429993777225,
+      "grad_norm": 1.7602269649505615,
+      "learning_rate": 1.141078838174274e-06,
+      "loss": 2.4316,
+      "step": 69
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "grad_norm": 1.7635431289672852,
+      "learning_rate": 1.161825726141079e-06,
+      "loss": 2.4674,
+      "step": 70
+    },
+    {
+      "epoch": 0.08711885500933417,
+      "eval_loss": 2.4922080039978027,
+      "eval_runtime": 45.3028,
+      "eval_samples_per_second": 22.074,
+      "eval_steps_per_second": 0.927,
+      "step": 70
+    },
+    {
+      "epoch": 0.08836341008089608,
+      "grad_norm": 2.0913803577423096,
+      "learning_rate": 1.182572614107884e-06,
+      "loss": 2.4341,
+      "step": 71
+    },
+    {
+      "epoch": 0.089607965152458,
+      "grad_norm": 1.8292183876037598,
+      "learning_rate": 1.2033195020746888e-06,
+      "loss": 2.4291,
+      "step": 72
+    },
+    {
+      "epoch": 0.09085252022401992,
+      "grad_norm": 2.206770181655884,
+      "learning_rate": 1.224066390041494e-06,
+      "loss": 2.3764,
+      "step": 73
+    },
+    {
+      "epoch": 0.09209707529558182,
+      "grad_norm": 1.8263559341430664,
+      "learning_rate": 1.2448132780082988e-06,
+      "loss": 2.4082,
+      "step": 74
+    },
+    {
+      "epoch": 0.09334163036714374,
+      "grad_norm": 1.9405455589294434,
+      "learning_rate": 1.2655601659751037e-06,
+      "loss": 2.4673,
+      "step": 75
+    },
+    {
+      "epoch": 0.09458618543870566,
+      "grad_norm": 2.138108015060425,
+      "learning_rate": 1.2863070539419086e-06,
+      "loss": 2.3879,
+      "step": 76
+    },
+    {
+      "epoch": 0.09583074051026758,
+      "grad_norm": 1.9024745225906372,
+      "learning_rate": 1.307053941908714e-06,
+      "loss": 2.4344,
+      "step": 77
+    },
+    {
+      "epoch": 0.0970752955818295,
+      "grad_norm": 1.9136689901351929,
+      "learning_rate": 1.3278008298755188e-06,
+      "loss": 2.474,
+      "step": 78
+    },
+    {
+      "epoch": 0.09831985065339141,
+      "grad_norm": 2.4752795696258545,
+      "learning_rate": 1.3485477178423237e-06,
+      "loss": 2.3412,
+      "step": 79
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "grad_norm": 1.8720006942749023,
+      "learning_rate": 1.3692946058091288e-06,
+      "loss": 2.3438,
+      "step": 80
+    },
+    {
+      "epoch": 0.09956440572495333,
+      "eval_loss": 2.3980512619018555,
+      "eval_runtime": 48.2788,
+      "eval_samples_per_second": 20.713,
+      "eval_steps_per_second": 0.87,
+      "step": 80
+    },
+    {
+      "epoch": 0.10080896079651525,
+      "grad_norm": 2.671691656112671,
+      "learning_rate": 1.3900414937759337e-06,
+      "loss": 2.3336,
+      "step": 81
+    },
+    {
+      "epoch": 0.10205351586807716,
+      "grad_norm": 2.2953391075134277,
+      "learning_rate": 1.4107883817427386e-06,
+      "loss": 2.377,
+      "step": 82
+    },
+    {
+      "epoch": 0.10329807093963908,
+      "grad_norm": 3.009018898010254,
+      "learning_rate": 1.4315352697095435e-06,
+      "loss": 2.2977,
+      "step": 83
+    },
+    {
+      "epoch": 0.104542626011201,
+      "grad_norm": 2.664454936981201,
+      "learning_rate": 1.4522821576763488e-06,
+      "loss": 2.3271,
+      "step": 84
+    },
+    {
+      "epoch": 0.10578718108276292,
+      "grad_norm": 3.017303705215454,
+      "learning_rate": 1.4730290456431537e-06,
+      "loss": 2.3251,
+      "step": 85
+    },
+    {
+      "epoch": 0.10703173615432483,
+      "grad_norm": 2.634716510772705,
+      "learning_rate": 1.4937759336099586e-06,
+      "loss": 2.332,
+      "step": 86
+    },
+    {
+      "epoch": 0.10827629122588675,
+      "grad_norm": 3.059644937515259,
+      "learning_rate": 1.5145228215767635e-06,
+      "loss": 2.3478,
+      "step": 87
+    },
+    {
+      "epoch": 0.10952084629744867,
+      "grad_norm": 2.6962637901306152,
+      "learning_rate": 1.5352697095435686e-06,
+      "loss": 2.2792,
+      "step": 88
+    },
+    {
+      "epoch": 0.11076540136901059,
+      "grad_norm": 3.419729709625244,
+      "learning_rate": 1.5560165975103735e-06,
+      "loss": 2.2571,
+      "step": 89
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "grad_norm": 2.740781545639038,
+      "learning_rate": 1.5767634854771784e-06,
+      "loss": 2.2875,
+      "step": 90
+    },
+    {
+      "epoch": 0.11200995644057249,
+      "eval_loss": 2.30843186378479,
+      "eval_runtime": 49.9264,
+      "eval_samples_per_second": 20.029,
+      "eval_steps_per_second": 0.841,
+      "step": 90
+    },
+    {
+      "epoch": 0.1132545115121344,
+      "grad_norm": 2.5608789920806885,
+      "learning_rate": 1.5975103734439833e-06,
+      "loss": 2.2592,
+      "step": 91
+    },
+    {
+      "epoch": 0.11449906658369632,
+      "grad_norm": 2.687999963760376,
+      "learning_rate": 1.6182572614107886e-06,
+      "loss": 2.2546,
+      "step": 92
+    },
+    {
+      "epoch": 0.11574362165525824,
+      "grad_norm": 2.695909023284912,
+      "learning_rate": 1.6390041493775935e-06,
+      "loss": 2.2525,
+      "step": 93
+    },
+    {
+      "epoch": 0.11698817672682016,
+      "grad_norm": 2.818357467651367,
+      "learning_rate": 1.6597510373443984e-06,
+      "loss": 2.216,
+      "step": 94
+    },
+    {
+      "epoch": 0.11823273179838208,
+      "grad_norm": 2.884119987487793,
+      "learning_rate": 1.6804979253112035e-06,
+      "loss": 2.2321,
+      "step": 95
+    },
+    {
+      "epoch": 0.119477286869944,
+      "grad_norm": 2.52104115486145,
+      "learning_rate": 1.7012448132780084e-06,
+      "loss": 2.199,
+      "step": 96
+    },
+    {
+      "epoch": 0.12072184194150591,
+      "grad_norm": 2.420313596725464,
+      "learning_rate": 1.7219917012448133e-06,
+      "loss": 2.1862,
+      "step": 97
+    },
+    {
+      "epoch": 0.12196639701306783,
+      "grad_norm": 2.8047542572021484,
+      "learning_rate": 1.7427385892116182e-06,
+      "loss": 2.1793,
+      "step": 98
+    },
+    {
+      "epoch": 0.12321095208462975,
+      "grad_norm": 2.836482286453247,
+      "learning_rate": 1.7634854771784235e-06,
+      "loss": 2.2271,
+      "step": 99
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "grad_norm": 2.5282301902770996,
+      "learning_rate": 1.7842323651452284e-06,
+      "loss": 2.1768,
+      "step": 100
+    },
+    {
+      "epoch": 0.12445550715619166,
+      "eval_loss": 2.2312686443328857,
+      "eval_runtime": 49.2382,
+      "eval_samples_per_second": 20.309,
+      "eval_steps_per_second": 0.853,
+      "step": 100
+    },
+    {
+      "epoch": 0.12570006222775357,
+      "grad_norm": 3.3407280445098877,
+      "learning_rate": 1.8049792531120333e-06,
+      "loss": 2.1666,
+      "step": 101
+    },
+    {
+      "epoch": 0.1269446172993155,
+      "grad_norm": 2.4754133224487305,
+      "learning_rate": 1.8257261410788382e-06,
+      "loss": 2.1768,
+      "step": 102
+    },
+    {
+      "epoch": 0.1281891723708774,
+      "grad_norm": 3.430889129638672,
+      "learning_rate": 1.8464730290456433e-06,
+      "loss": 2.1953,
+      "step": 103
+    },
+    {
+      "epoch": 0.12943372744243933,
+      "grad_norm": 2.835294246673584,
+      "learning_rate": 1.8672199170124482e-06,
+      "loss": 2.146,
+      "step": 104
+    },
+    {
+      "epoch": 0.13067828251400124,
+      "grad_norm": 3.1532323360443115,
+      "learning_rate": 1.8879668049792531e-06,
+      "loss": 2.1729,
+      "step": 105
+    },
+    {
+      "epoch": 0.13192283758556317,
+      "grad_norm": 3.2278342247009277,
+      "learning_rate": 1.908713692946058e-06,
+      "loss": 2.1336,
+      "step": 106
+    },
+    {
+      "epoch": 0.13316739265712507,
+      "grad_norm": 2.7892515659332275,
+      "learning_rate": 1.929460580912863e-06,
+      "loss": 2.141,
+      "step": 107
+    },
+    {
+      "epoch": 0.134411947728687,
+      "grad_norm": 3.0179977416992188,
+      "learning_rate": 1.9502074688796682e-06,
+      "loss": 2.1255,
+      "step": 108
+    },
+    {
+      "epoch": 0.1356565028002489,
+      "grad_norm": 2.977935552597046,
+      "learning_rate": 1.970954356846473e-06,
+      "loss": 2.0986,
+      "step": 109
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "grad_norm": 3.3042492866516113,
+      "learning_rate": 1.991701244813278e-06,
+      "loss": 2.0992,
+      "step": 110
+    },
+    {
+      "epoch": 0.13690105787181084,
+      "eval_loss": 2.171299695968628,
+      "eval_runtime": 42.9511,
+      "eval_samples_per_second": 23.282,
+      "eval_steps_per_second": 0.978,
+      "step": 110
+    },
+    {
+      "epoch": 0.13814561294337274,
+      "grad_norm": 2.9750890731811523,
+      "learning_rate": 2.012448132780083e-06,
+      "loss": 2.1116,
+      "step": 111
+    },
+    {
+      "epoch": 0.13939016801493467,
+      "grad_norm": 2.8199286460876465,
+      "learning_rate": 2.0331950207468883e-06,
+      "loss": 2.0822,
+      "step": 112
+    },
+    {
+      "epoch": 0.14063472308649658,
+      "grad_norm": 2.930532217025757,
+      "learning_rate": 2.053941908713693e-06,
+      "loss": 2.161,
+      "step": 113
+    },
+    {
+      "epoch": 0.14187927815805848,
+      "grad_norm": 3.3400321006774902,
+      "learning_rate": 2.074688796680498e-06,
+      "loss": 2.0953,
+      "step": 114
+    },
+    {
+      "epoch": 0.1431238332296204,
+      "grad_norm": 2.716564655303955,
+      "learning_rate": 2.095435684647303e-06,
+      "loss": 2.0757,
+      "step": 115
+    },
+    {
+      "epoch": 0.14436838830118232,
+      "grad_norm": 2.7586236000061035,
+      "learning_rate": 2.116182572614108e-06,
+      "loss": 2.0979,
+      "step": 116
+    },
+    {
+      "epoch": 0.14561294337274425,
+      "grad_norm": 2.622126340866089,
+      "learning_rate": 2.136929460580913e-06,
+      "loss": 2.0748,
+      "step": 117
+    },
+    {
+      "epoch": 0.14685749844430615,
+      "grad_norm": 2.646477460861206,
+      "learning_rate": 2.157676348547718e-06,
+      "loss": 2.0605,
+      "step": 118
+    },
+    {
+      "epoch": 0.14810205351586808,
+      "grad_norm": 2.8882410526275635,
+      "learning_rate": 2.178423236514523e-06,
+      "loss": 2.0501,
+      "step": 119
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "grad_norm": 2.8302247524261475,
+      "learning_rate": 2.199170124481328e-06,
+      "loss": 2.0592,
+      "step": 120
+    },
+    {
+      "epoch": 0.14934660858742999,
+      "eval_loss": 2.1291966438293457,
+      "eval_runtime": 47.5399,
+      "eval_samples_per_second": 21.035,
+      "eval_steps_per_second": 0.883,
+      "step": 120
+    },
+    {
+      "epoch": 0.15059116365899192,
+      "grad_norm": 3.26979660987854,
+      "learning_rate": 2.219917012448133e-06,
+      "loss": 2.0422,
+      "step": 121
+    },
+    {
+      "epoch": 0.15183571873055382,
+      "grad_norm": 2.7483913898468018,
+      "learning_rate": 2.240663900414938e-06,
+      "loss": 2.0872,
+      "step": 122
+    },
+    {
+      "epoch": 0.15308027380211575,
+      "grad_norm": 3.1252024173736572,
+      "learning_rate": 2.2614107883817427e-06,
+      "loss": 2.0684,
+      "step": 123
+    },
+    {
+      "epoch": 0.15432482887367766,
+      "grad_norm": 2.9175283908843994,
+      "learning_rate": 2.282157676348548e-06,
+      "loss": 2.0522,
+      "step": 124
+    },
+    {
+      "epoch": 0.1555693839452396,
+      "grad_norm": 2.7950754165649414,
+      "learning_rate": 2.302904564315353e-06,
+      "loss": 2.065,
+      "step": 125
+    },
+    {
+      "epoch": 0.1568139390168015,
+      "grad_norm": 2.84049654006958,
+      "learning_rate": 2.323651452282158e-06,
+      "loss": 2.0781,
+      "step": 126
+    },
+    {
+      "epoch": 0.15805849408836342,
+      "grad_norm": 2.7396647930145264,
+      "learning_rate": 2.3443983402489627e-06,
+      "loss": 2.1758,
+      "step": 127
+    },
+    {
+      "epoch": 0.15930304915992533,
+      "grad_norm": 3.0765926837921143,
+      "learning_rate": 2.365145228215768e-06,
+      "loss": 2.0172,
+      "step": 128
+    },
+    {
+      "epoch": 0.16054760423148726,
+      "grad_norm": 3.217189073562622,
+      "learning_rate": 2.385892116182573e-06,
+      "loss": 2.0286,
+      "step": 129
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "grad_norm": 3.141545295715332,
+      "learning_rate": 2.4066390041493776e-06,
+      "loss": 2.0583,
+      "step": 130
+    },
+    {
+      "epoch": 0.16179215930304916,
+      "eval_loss": 2.0946149826049805,
+      "eval_runtime": 54.92,
+      "eval_samples_per_second": 18.208,
+      "eval_steps_per_second": 0.765,
+      "step": 130
+    },
+    {
+      "epoch": 0.16303671437461106,
+      "grad_norm": 2.850052833557129,
+      "learning_rate": 2.4273858921161828e-06,
+      "loss": 2.0746,
+      "step": 131
+    },
+    {
+      "epoch": 0.164281269446173,
+      "grad_norm": 3.28913950920105,
+      "learning_rate": 2.448132780082988e-06,
+      "loss": 2.0539,
+      "step": 132
+    },
+    {
+      "epoch": 0.1655258245177349,
+      "grad_norm": 2.8819124698638916,
+      "learning_rate": 2.468879668049793e-06,
+      "loss": 1.9843,
+      "step": 133
+    },
+    {
+      "epoch": 0.16677037958929683,
+      "grad_norm": 3.6254632472991943,
+      "learning_rate": 2.4896265560165977e-06,
+      "loss": 2.0233,
+      "step": 134
+    },
+    {
+      "epoch": 0.16801493466085873,
+      "grad_norm": 2.7385146617889404,
+      "learning_rate": 2.5103734439834028e-06,
+      "loss": 1.9667,
+      "step": 135
+    },
+    {
+      "epoch": 0.16925948973242066,
+      "grad_norm": 2.9722647666931152,
+      "learning_rate": 2.5311203319502074e-06,
+      "loss": 1.9527,
+      "step": 136
+    },
+    {
+      "epoch": 0.17050404480398257,
+      "grad_norm": 3.139526605606079,
+      "learning_rate": 2.5518672199170125e-06,
+      "loss": 1.9967,
+      "step": 137
+    },
+    {
+      "epoch": 0.1717485998755445,
+      "grad_norm": 3.399920701980591,
+      "learning_rate": 2.5726141078838172e-06,
+      "loss": 1.9798,
+      "step": 138
+    },
+    {
+      "epoch": 0.1729931549471064,
+      "grad_norm": 2.8017327785491943,
+      "learning_rate": 2.5933609958506228e-06,
+      "loss": 1.983,
+      "step": 139
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "grad_norm": 3.9967198371887207,
+      "learning_rate": 2.614107883817428e-06,
+      "loss": 1.977,
+      "step": 140
+    },
+    {
+      "epoch": 0.17423771001866833,
+      "eval_loss": 2.059037685394287,
+      "eval_runtime": 52.6232,
+      "eval_samples_per_second": 19.003,
+      "eval_steps_per_second": 0.798,
+      "step": 140
+    },
+    {
+      "epoch": 0.17548226509023024,
+      "grad_norm": 2.893092393875122,
+      "learning_rate": 2.6348547717842326e-06,
+      "loss": 1.9888,
+      "step": 141
+    },
+    {
+      "epoch": 0.17672682016179217,
+      "grad_norm": 3.634352207183838,
+      "learning_rate": 2.6556016597510377e-06,
+      "loss": 2.0726,
+      "step": 142
+    },
+    {
+      "epoch": 0.17797137523335407,
+      "grad_norm": 3.3651444911956787,
+      "learning_rate": 2.6763485477178423e-06,
+      "loss": 1.973,
+      "step": 143
+    },
+    {
+      "epoch": 0.179215930304916,
+      "grad_norm": 3.968986988067627,
+      "learning_rate": 2.6970954356846475e-06,
+      "loss": 1.9948,
+      "step": 144
+    },
+    {
+      "epoch": 0.1804604853764779,
+      "grad_norm": 3.20105242729187,
+      "learning_rate": 2.717842323651452e-06,
+      "loss": 1.9238,
+      "step": 145
+    },
+    {
+      "epoch": 0.18170504044803984,
+      "grad_norm": 3.648339033126831,
+      "learning_rate": 2.7385892116182577e-06,
+      "loss": 1.9923,
+      "step": 146
+    },
+    {
+      "epoch": 0.18294959551960174,
+      "grad_norm": 3.0856316089630127,
+      "learning_rate": 2.7593360995850628e-06,
+      "loss": 1.9749,
+      "step": 147
+    },
+    {
+      "epoch": 0.18419415059116365,
+      "grad_norm": 3.2681071758270264,
+      "learning_rate": 2.7800829875518675e-06,
+      "loss": 1.9304,
+      "step": 148
+    },
+    {
+      "epoch": 0.18543870566272558,
+      "grad_norm": 2.634958267211914,
+      "learning_rate": 2.8008298755186726e-06,
+      "loss": 1.9237,
+      "step": 149
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "grad_norm": 2.769491672515869,
+      "learning_rate": 2.8215767634854773e-06,
+      "loss": 1.8963,
+      "step": 150
+    },
+    {
+      "epoch": 0.18668326073428748,
+      "eval_loss": 2.0394654273986816,
+      "eval_runtime": 55.2136,
+      "eval_samples_per_second": 18.111,
+      "eval_steps_per_second": 0.761,
+      "step": 150
+    },
+    {
+      "epoch": 0.1879278158058494,
+      "grad_norm": 3.4345781803131104,
+      "learning_rate": 2.8423236514522824e-06,
+      "loss": 1.9836,
+      "step": 151
+    },
+    {
+      "epoch": 0.18917237087741132,
+      "grad_norm": 3.115727424621582,
+      "learning_rate": 2.863070539419087e-06,
+      "loss": 1.9352,
+      "step": 152
+    },
+    {
+      "epoch": 0.19041692594897325,
+      "grad_norm": 3.050652503967285,
+      "learning_rate": 2.883817427385892e-06,
+      "loss": 1.9031,
+      "step": 153
+    },
+    {
+      "epoch": 0.19166148102053515,
+      "grad_norm": 2.9404428005218506,
+      "learning_rate": 2.9045643153526977e-06,
+      "loss": 1.9196,
+      "step": 154
+    },
+    {
+      "epoch": 0.19290603609209708,
+      "grad_norm": 3.080810785293579,
+      "learning_rate": 2.9253112033195024e-06,
+      "loss": 1.9405,
+      "step": 155
+    },
+    {
+      "epoch": 0.194150591163659,
+      "grad_norm": 3.058558702468872,
+      "learning_rate": 2.9460580912863075e-06,
+      "loss": 1.9052,
+      "step": 156
+    },
+    {
+      "epoch": 0.19539514623522092,
+      "grad_norm": 3.307955026626587,
+      "learning_rate": 2.966804979253112e-06,
+      "loss": 1.9245,
+      "step": 157
+    },
+    {
+      "epoch": 0.19663970130678282,
+      "grad_norm": 2.845506191253662,
+      "learning_rate": 2.9875518672199173e-06,
+      "loss": 1.9461,
+      "step": 158
+    },
+    {
+      "epoch": 0.19788425637834475,
+      "grad_norm": 2.758654832839966,
+      "learning_rate": 3.008298755186722e-06,
+      "loss": 1.8885,
+      "step": 159
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "grad_norm": 3.161252737045288,
+      "learning_rate": 3.029045643153527e-06,
+      "loss": 1.8535,
+      "step": 160
+    },
+    {
+      "epoch": 0.19912881144990666,
+      "eval_loss": 2.018101453781128,
+      "eval_runtime": 51.0744,
+      "eval_samples_per_second": 19.579,
+      "eval_steps_per_second": 0.822,
+      "step": 160
+    },
+    {
+      "epoch": 0.2003733665214686,
+      "grad_norm": 3.196847677230835,
+      "learning_rate": 3.0497925311203326e-06,
+      "loss": 1.8853,
+      "step": 161
+    },
+    {
+      "epoch": 0.2016179215930305,
+      "grad_norm": 2.8364663124084473,
+      "learning_rate": 3.0705394190871373e-06,
+      "loss": 1.9463,
+      "step": 162
+    },
+    {
+      "epoch": 0.2028624766645924,
+      "grad_norm": 3.0874054431915283,
+      "learning_rate": 3.0912863070539424e-06,
+      "loss": 1.9254,
+      "step": 163
+    },
+    {
+      "epoch": 0.20410703173615433,
+      "grad_norm": 2.7914493083953857,
+      "learning_rate": 3.112033195020747e-06,
+      "loss": 1.9213,
+      "step": 164
+    },
+    {
+      "epoch": 0.20535158680771623,
+      "grad_norm": 3.3871428966522217,
+      "learning_rate": 3.132780082987552e-06,
+      "loss": 1.8993,
+      "step": 165
+    },
+    {
+      "epoch": 0.20659614187927816,
+      "grad_norm": 3.096653461456299,
+      "learning_rate": 3.153526970954357e-06,
+      "loss": 1.8827,
+      "step": 166
+    },
+    {
+      "epoch": 0.20784069695084006,
+      "grad_norm": 2.8276076316833496,
+      "learning_rate": 3.174273858921162e-06,
+      "loss": 1.912,
+      "step": 167
+    },
+    {
+      "epoch": 0.209085252022402,
+      "grad_norm": 3.3058435916900635,
+      "learning_rate": 3.1950207468879666e-06,
+      "loss": 1.8381,
+      "step": 168
+    },
+    {
+      "epoch": 0.2103298070939639,
+      "grad_norm": 3.5017333030700684,
+      "learning_rate": 3.215767634854772e-06,
+      "loss": 1.9544,
+      "step": 169
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "grad_norm": 3.457296133041382,
+      "learning_rate": 3.2365145228215773e-06,
+      "loss": 1.922,
+      "step": 170
+    },
+    {
+      "epoch": 0.21157436216552583,
+      "eval_loss": 1.988455891609192,
+      "eval_runtime": 56.2099,
+      "eval_samples_per_second": 17.79,
+      "eval_steps_per_second": 0.747,
+      "step": 170
+    },
+    {
+      "epoch": 0.21281891723708773,
+      "grad_norm": 3.1964566707611084,
+      "learning_rate": 3.257261410788382e-06,
+      "loss": 1.8925,
+      "step": 171
+    },
+    {
+      "epoch": 0.21406347230864967,
+      "grad_norm": 3.234652042388916,
+      "learning_rate": 3.278008298755187e-06,
+      "loss": 1.884,
+      "step": 172
+    },
+    {
+      "epoch": 0.21530802738021157,
+      "grad_norm": 3.14414119720459,
+      "learning_rate": 3.2987551867219918e-06,
+      "loss": 1.8905,
+      "step": 173
+    },
+    {
+      "epoch": 0.2165525824517735,
+      "grad_norm": 3.606379508972168,
+      "learning_rate": 3.319502074688797e-06,
+      "loss": 1.946,
+      "step": 174
+    },
+    {
+      "epoch": 0.2177971375233354,
+      "grad_norm": 2.986646890640259,
+      "learning_rate": 3.3402489626556016e-06,
+      "loss": 1.8691,
+      "step": 175
+    },
+    {
+      "epoch": 0.21904169259489734,
+      "grad_norm": 3.7298269271850586,
+      "learning_rate": 3.360995850622407e-06,
+      "loss": 1.8717,
+      "step": 176
+    },
+    {
+      "epoch": 0.22028624766645924,
+      "grad_norm": 3.422295093536377,
+      "learning_rate": 3.381742738589212e-06,
+      "loss": 1.8568,
+      "step": 177
+    },
+    {
+      "epoch": 0.22153080273802117,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.8471,
+      "step": 178
+    },
+    {
+      "epoch": 0.22277535780958307,
+      "grad_norm": 3.2293782234191895,
+      "learning_rate": 3.402489626556017e-06,
+      "loss": 1.9303,
+      "step": 179
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "grad_norm": 3.506223440170288,
+      "learning_rate": 3.423236514522822e-06,
+      "loss": 1.8237,
+      "step": 180
+    },
+    {
+      "epoch": 0.22401991288114498,
+      "eval_loss": 1.9734643697738647,
+      "eval_runtime": 52.2815,
+      "eval_samples_per_second": 19.127,
+      "eval_steps_per_second": 0.803,
+      "step": 180
+    },
+    {
+      "epoch": 0.2252644679527069,
+      "grad_norm": 3.0156619548797607,
+      "learning_rate": 3.4439834024896267e-06,
+      "loss": 1.908,
+      "step": 181
+    },
+    {
+      "epoch": 0.2265090230242688,
+      "grad_norm": 3.55517315864563,
+      "learning_rate": 3.4647302904564318e-06,
+      "loss": 1.9104,
+      "step": 182
+    },
+    {
+      "epoch": 0.22775357809583074,
+      "grad_norm": 3.144984006881714,
+      "learning_rate": 3.4854771784232365e-06,
+      "loss": 1.8198,
+      "step": 183
+    },
+    {
+      "epoch": 0.22899813316739265,
+      "grad_norm": 4.302074432373047,
+      "learning_rate": 3.5062240663900416e-06,
+      "loss": 1.8237,
+      "step": 184
+    },
+    {
+      "epoch": 0.23024268823895458,
+      "grad_norm": 3.0522446632385254,
+      "learning_rate": 3.526970954356847e-06,
+      "loss": 1.8409,
+      "step": 185
+    },
+    {
+      "epoch": 0.23148724331051648,
+      "grad_norm": 3.8607394695281982,
+      "learning_rate": 3.5477178423236518e-06,
+      "loss": 1.8807,
+      "step": 186
+    },
+    {
+      "epoch": 0.23273179838207841,
+      "grad_norm": 2.9236302375793457,
+      "learning_rate": 3.568464730290457e-06,
+      "loss": 1.7873,
+      "step": 187
+    },
+    {
+      "epoch": 0.23397635345364032,
+      "grad_norm": 4.013780117034912,
+      "learning_rate": 3.5892116182572616e-06,
+      "loss": 1.7909,
+      "step": 188
+    },
+    {
+      "epoch": 0.23522090852520225,
+      "grad_norm": 3.0933122634887695,
+      "learning_rate": 3.6099585062240667e-06,
+      "loss": 1.8469,
+      "step": 189
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "grad_norm": 3.487816095352173,
+      "learning_rate": 3.6307053941908714e-06,
+      "loss": 1.8469,
+      "step": 190
+    },
+    {
+      "epoch": 0.23646546359676415,
+      "eval_loss": 1.9604240655899048,
+      "eval_runtime": 53.4604,
+      "eval_samples_per_second": 18.705,
+      "eval_steps_per_second": 0.786,
+      "step": 190
+    },
+    {
+      "epoch": 0.23771001866832608,
+      "grad_norm": 2.829219341278076,
+      "learning_rate": 3.6514522821576765e-06,
+      "loss": 1.8639,
+      "step": 191
+    },
+    {
+      "epoch": 0.238954573739888,
+      "grad_norm": 3.595534324645996,
+      "learning_rate": 3.672199170124482e-06,
+      "loss": 1.8988,
+      "step": 192
+    },
+    {
+      "epoch": 0.24019912881144992,
+      "grad_norm": 2.9022483825683594,
+      "learning_rate": 3.6929460580912867e-06,
+      "loss": 1.8972,
+      "step": 193
+    },
+    {
+      "epoch": 0.24144368388301182,
+      "grad_norm": 3.0667994022369385,
+      "learning_rate": 3.713692946058092e-06,
+      "loss": 1.8499,
+      "step": 194
+    },
+    {
+      "epoch": 0.24268823895457373,
+      "grad_norm": 3.0498485565185547,
+      "learning_rate": 3.7344398340248965e-06,
+      "loss": 1.8341,
+      "step": 195
+    },
+    {
+      "epoch": 0.24393279402613566,
+      "grad_norm": 4.058084964752197,
+      "learning_rate": 3.7551867219917016e-06,
+      "loss": 1.8641,
+      "step": 196
+    },
+    {
+      "epoch": 0.24517734909769756,
+      "grad_norm": 2.9995028972625732,
+      "learning_rate": 3.7759336099585063e-06,
+      "loss": 1.7885,
+      "step": 197
+    },
+    {
+      "epoch": 0.2464219041692595,
+      "grad_norm": 3.254704475402832,
+      "learning_rate": 3.7966804979253114e-06,
+      "loss": 1.7544,
+      "step": 198
+    },
+    {
+      "epoch": 0.2476664592408214,
+      "grad_norm": 2.9126744270324707,
+      "learning_rate": 3.817427385892116e-06,
+      "loss": 1.8461,
+      "step": 199
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "grad_norm": 3.475247621536255,
+      "learning_rate": 3.838174273858922e-06,
+      "loss": 1.8095,
+      "step": 200
+    },
+    {
+      "epoch": 0.24891101431238333,
+      "eval_loss": 1.9373760223388672,
+      "eval_runtime": 51.8733,
+      "eval_samples_per_second": 19.278,
+      "eval_steps_per_second": 0.81,
+      "step": 200
+    },
+    {
+      "epoch": 0.25015556938394523,
+      "grad_norm": 3.2663731575012207,
+      "learning_rate": 3.858921161825726e-06,
+      "loss": 1.8076,
+      "step": 201
+    },
+    {
+      "epoch": 0.25140012445550713,
+      "grad_norm": 3.1151726245880127,
+      "learning_rate": 3.879668049792531e-06,
+      "loss": 1.8143,
+      "step": 202
+    },
+    {
+      "epoch": 0.2526446795270691,
+      "grad_norm": 3.102038860321045,
+      "learning_rate": 3.9004149377593365e-06,
+      "loss": 1.8279,
+      "step": 203
+    },
+    {
+      "epoch": 0.253889234598631,
+      "grad_norm": 3.370642900466919,
+      "learning_rate": 3.921161825726142e-06,
+      "loss": 1.8522,
+      "step": 204
+    },
+    {
+      "epoch": 0.2551337896701929,
+      "grad_norm": 3.116128921508789,
+      "learning_rate": 3.941908713692946e-06,
+      "loss": 1.8104,
+      "step": 205
+    },
+    {
+      "epoch": 0.2563783447417548,
+      "grad_norm": 3.12028431892395,
+      "learning_rate": 3.962655601659751e-06,
+      "loss": 1.7723,
+      "step": 206
+    },
+    {
+      "epoch": 0.25762289981331676,
+      "grad_norm": 3.1251418590545654,
+      "learning_rate": 3.983402489626556e-06,
+      "loss": 1.863,
+      "step": 207
+    },
+    {
+      "epoch": 0.25886745488487867,
+      "grad_norm": 3.073702812194824,
+      "learning_rate": 4.004149377593361e-06,
+      "loss": 1.805,
+      "step": 208
+    },
+    {
+      "epoch": 0.26011200995644057,
+      "grad_norm": 2.9772469997406006,
+      "learning_rate": 4.024896265560166e-06,
+      "loss": 1.8251,
+      "step": 209
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "grad_norm": 2.9970037937164307,
+      "learning_rate": 4.045643153526971e-06,
+      "loss": 1.8386,
+      "step": 210
+    },
+    {
+      "epoch": 0.2613565650280025,
+      "eval_loss": 1.9167065620422363,
+      "eval_runtime": 42.3328,
+      "eval_samples_per_second": 23.622,
+      "eval_steps_per_second": 0.992,
+      "step": 210
+    },
+    {
+      "epoch": 0.26260112009956443,
+      "grad_norm": 3.1391561031341553,
+      "learning_rate": 4.0663900414937765e-06,
+      "loss": 1.8109,
+      "step": 211
+    },
+    {
+      "epoch": 0.26384567517112634,
+      "grad_norm": 3.161538600921631,
+      "learning_rate": 4.087136929460581e-06,
+      "loss": 1.8441,
+      "step": 212
+    },
+    {
+      "epoch": 0.26509023024268824,
+      "grad_norm": 3.12412166595459,
+      "learning_rate": 4.107883817427386e-06,
+      "loss": 1.7736,
+      "step": 213
+    },
+    {
+      "epoch": 0.26633478531425014,
+      "grad_norm": 3.3241145610809326,
+      "learning_rate": 4.128630705394191e-06,
+      "loss": 1.8067,
+      "step": 214
+    },
+    {
+      "epoch": 0.26757934038581205,
+      "grad_norm": 3.503307819366455,
+      "learning_rate": 4.149377593360996e-06,
+      "loss": 1.7858,
+      "step": 215
+    },
+    {
+      "epoch": 0.268823895457374,
+      "grad_norm": 3.1009578704833984,
+      "learning_rate": 4.170124481327801e-06,
+      "loss": 1.7542,
+      "step": 216
+    },
+    {
+      "epoch": 0.2700684505289359,
+      "grad_norm": 3.2236011028289795,
+      "learning_rate": 4.190871369294606e-06,
+      "loss": 1.8006,
+      "step": 217
+    },
+    {
+      "epoch": 0.2713130056004978,
+      "grad_norm": 3.048935651779175,
+      "learning_rate": 4.211618257261411e-06,
+      "loss": 1.7706,
+      "step": 218
+    },
+    {
+      "epoch": 0.2725575606720597,
+      "grad_norm": 3.3416147232055664,
+      "learning_rate": 4.232365145228216e-06,
+      "loss": 1.8108,
+      "step": 219
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "grad_norm": 3.2010738849639893,
+      "learning_rate": 4.253112033195021e-06,
+      "loss": 1.8561,
+      "step": 220
+    },
+    {
+      "epoch": 0.2738021157436217,
+      "eval_loss": 1.9111930131912231,
+      "eval_runtime": 50.4077,
+      "eval_samples_per_second": 19.838,
+      "eval_steps_per_second": 0.833,
+      "step": 220
+    },
+    {
+      "epoch": 0.2750466708151836,
+      "grad_norm": 3.4033374786376953,
+      "learning_rate": 4.273858921161826e-06,
+      "loss": 1.8218,
+      "step": 221
+    },
+    {
+      "epoch": 0.2762912258867455,
+      "grad_norm": 3.2086002826690674,
+      "learning_rate": 4.294605809128631e-06,
+      "loss": 1.8213,
+      "step": 222
+    },
+    {
+      "epoch": 0.2775357809583074,
+      "grad_norm": 4.090956211090088,
+      "learning_rate": 4.315352697095436e-06,
+      "loss": 1.8455,
+      "step": 223
+    },
+    {
+      "epoch": 0.27878033602986935,
+      "grad_norm": 3.1142985820770264,
+      "learning_rate": 4.336099585062241e-06,
+      "loss": 1.7737,
+      "step": 224
+    },
+    {
+      "epoch": 0.28002489110143125,
+      "grad_norm": 3.369669198989868,
+      "learning_rate": 4.356846473029046e-06,
+      "loss": 1.824,
+      "step": 225
+    },
+    {
+      "epoch": 0.28126944617299315,
+      "grad_norm": 3.165672779083252,
+      "learning_rate": 4.3775933609958506e-06,
+      "loss": 1.7768,
+      "step": 226
+    },
+    {
+      "epoch": 0.28251400124455506,
+      "grad_norm": 3.3343470096588135,
+      "learning_rate": 4.398340248962656e-06,
+      "loss": 1.7278,
+      "step": 227
+    },
+    {
+      "epoch": 0.28375855631611696,
+      "grad_norm": 3.2574994564056396,
+      "learning_rate": 4.419087136929461e-06,
+      "loss": 1.8439,
+      "step": 228
+    },
+    {
+      "epoch": 0.2850031113876789,
+      "grad_norm": 3.043928623199463,
+      "learning_rate": 4.439834024896266e-06,
+      "loss": 1.818,
+      "step": 229
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "grad_norm": 2.9703125953674316,
+      "learning_rate": 4.460580912863071e-06,
+      "loss": 1.773,
+      "step": 230
+    },
+    {
+      "epoch": 0.2862476664592408,
+      "eval_loss": 1.898109793663025,
+      "eval_runtime": 45.6023,
+      "eval_samples_per_second": 21.929,
+      "eval_steps_per_second": 0.921,
+      "step": 230
+    },
+    {
+      "epoch": 0.2874922215308027,
+      "grad_norm": 2.9997220039367676,
+      "learning_rate": 4.481327800829876e-06,
+      "loss": 1.7727,
+      "step": 231
+    },
+    {
+      "epoch": 0.28873677660236463,
+      "grad_norm": 3.1358482837677,
+      "learning_rate": 4.502074688796681e-06,
+      "loss": 1.7764,
+      "step": 232
+    },
+    {
+      "epoch": 0.2899813316739266,
+      "grad_norm": 3.395747661590576,
+      "learning_rate": 4.5228215767634855e-06,
+      "loss": 1.7826,
+      "step": 233
+    },
+    {
+      "epoch": 0.2912258867454885,
+      "grad_norm": 3.7494754791259766,
+      "learning_rate": 4.543568464730291e-06,
+      "loss": 1.7181,
+      "step": 234
+    },
+    {
+      "epoch": 0.2924704418170504,
+      "grad_norm": 3.42293381690979,
+      "learning_rate": 4.564315352697096e-06,
+      "loss": 1.7773,
+      "step": 235
+    },
+    {
+      "epoch": 0.2937149968886123,
+      "grad_norm": 3.2524514198303223,
+      "learning_rate": 4.585062240663901e-06,
+      "loss": 1.7294,
+      "step": 236
+    },
+    {
+      "epoch": 0.29495955196017426,
+      "grad_norm": 3.657869577407837,
+      "learning_rate": 4.605809128630706e-06,
+      "loss": 1.762,
+      "step": 237
+    },
+    {
+      "epoch": 0.29620410703173616,
+      "grad_norm": 3.127372980117798,
+      "learning_rate": 4.626556016597511e-06,
+      "loss": 1.7623,
+      "step": 238
+    },
+    {
+      "epoch": 0.29744866210329807,
+      "grad_norm": 4.166962146759033,
+      "learning_rate": 4.647302904564316e-06,
+      "loss": 1.6995,
+      "step": 239
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "grad_norm": 3.094264030456543,
+      "learning_rate": 4.66804979253112e-06,
+      "loss": 1.7469,
+      "step": 240
+    },
+    {
+      "epoch": 0.29869321717485997,
+      "eval_loss": 1.8908178806304932,
+      "eval_runtime": 44.6332,
+      "eval_samples_per_second": 22.405,
+      "eval_steps_per_second": 0.941,
+      "step": 240
+    },
+    {
+      "epoch": 0.29993777224642193,
+      "grad_norm": 3.8644745349884033,
+      "learning_rate": 4.6887966804979255e-06,
+      "loss": 1.7644,
+      "step": 241
+    },
+    {
+      "epoch": 0.30118232731798383,
+      "grad_norm": 3.1488852500915527,
+      "learning_rate": 4.709543568464731e-06,
+      "loss": 1.772,
+      "step": 242
+    },
+    {
+      "epoch": 0.30242688238954574,
+      "grad_norm": 3.3179638385772705,
+      "learning_rate": 4.730290456431536e-06,
+      "loss": 1.7588,
+      "step": 243
+    },
+    {
+      "epoch": 0.30367143746110764,
+      "grad_norm": 3.328355550765991,
+      "learning_rate": 4.751037344398341e-06,
+      "loss": 1.7417,
+      "step": 244
+    },
+    {
+      "epoch": 0.30491599253266954,
+      "grad_norm": 3.2690482139587402,
+      "learning_rate": 4.771784232365146e-06,
+      "loss": 1.7562,
+      "step": 245
+    },
+    {
+      "epoch": 0.3061605476042315,
+      "grad_norm": 3.2759454250335693,
+      "learning_rate": 4.792531120331951e-06,
+      "loss": 1.7536,
+      "step": 246
+    },
+    {
+      "epoch": 0.3074051026757934,
+      "grad_norm": 2.937964916229248,
+      "learning_rate": 4.813278008298755e-06,
+      "loss": 1.7518,
+      "step": 247
+    },
+    {
+      "epoch": 0.3086496577473553,
+      "grad_norm": 3.0995302200317383,
+      "learning_rate": 4.83402489626556e-06,
+      "loss": 1.7593,
+      "step": 248
+    },
+    {
+      "epoch": 0.3098942128189172,
+      "grad_norm": 3.230459213256836,
+      "learning_rate": 4.8547717842323655e-06,
+      "loss": 1.6962,
+      "step": 249
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "grad_norm": 3.242576837539673,
+      "learning_rate": 4.875518672199171e-06,
+      "loss": 1.7307,
+      "step": 250
+    },
+    {
+      "epoch": 0.3111387678904792,
+      "eval_loss": 1.8805371522903442,
+      "eval_runtime": 45.9886,
+      "eval_samples_per_second": 21.745,
+      "eval_steps_per_second": 0.913,
+      "step": 250
+    },
+    {
+      "epoch": 0.3123833229620411,
+      "grad_norm": 3.097045421600342,
+      "learning_rate": 4.896265560165976e-06,
+      "loss": 1.7438,
+      "step": 251
+    },
+    {
+      "epoch": 0.313627878033603,
+      "grad_norm": 3.2428948879241943,
+      "learning_rate": 4.91701244813278e-06,
+      "loss": 1.7179,
+      "step": 252
+    },
+    {
+      "epoch": 0.3148724331051649,
+      "grad_norm": 3.196274518966675,
+      "learning_rate": 4.937759336099586e-06,
+      "loss": 1.7855,
+      "step": 253
+    },
+    {
+      "epoch": 0.31611698817672684,
+      "grad_norm": 2.978203535079956,
+      "learning_rate": 4.95850622406639e-06,
+      "loss": 1.7144,
+      "step": 254
+    },
+    {
+      "epoch": 0.31736154324828875,
+      "grad_norm": 3.2641701698303223,
+      "learning_rate": 4.979253112033195e-06,
+      "loss": 1.7427,
+      "step": 255
+    },
+    {
+      "epoch": 0.31860609831985065,
+      "grad_norm": 2.7441232204437256,
+      "learning_rate": 5e-06,
+      "loss": 1.7619,
+      "step": 256
+    },
+    {
+      "epoch": 0.31985065339141255,
+      "grad_norm": 3.0723495483398438,
+      "learning_rate": 4.99769372693727e-06,
+      "loss": 1.8017,
+      "step": 257
+    },
+    {
+      "epoch": 0.3210952084629745,
+      "grad_norm": 3.104752540588379,
+      "learning_rate": 4.995387453874539e-06,
+      "loss": 1.7761,
+      "step": 258
+    },
+    {
+      "epoch": 0.3223397635345364,
+      "grad_norm": 3.138627052307129,
+      "learning_rate": 4.993081180811809e-06,
+      "loss": 1.7879,
+      "step": 259
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "grad_norm": 2.8567333221435547,
+      "learning_rate": 4.990774907749078e-06,
+      "loss": 1.7561,
+      "step": 260
+    },
+    {
+      "epoch": 0.3235843186060983,
+      "eval_loss": 1.8606494665145874,
+      "eval_runtime": 46.453,
+      "eval_samples_per_second": 21.527,
+      "eval_steps_per_second": 0.904,
+      "step": 260
+    },
+    {
+      "epoch": 0.3248288736776602,
+      "grad_norm": 3.2696540355682373,
+      "learning_rate": 4.988468634686347e-06,
+      "loss": 1.7201,
+      "step": 261
+    },
+    {
+      "epoch": 0.3260734287492221,
+      "grad_norm": 2.6692731380462646,
+      "learning_rate": 4.986162361623617e-06,
+      "loss": 1.7264,
+      "step": 262
+    },
+    {
+      "epoch": 0.3273179838207841,
+      "grad_norm": 3.047549247741699,
+      "learning_rate": 4.983856088560886e-06,
+      "loss": 1.7362,
+      "step": 263
+    },
+    {
+      "epoch": 0.328562538892346,
+      "grad_norm": 3.0667457580566406,
+      "learning_rate": 4.981549815498156e-06,
+      "loss": 1.7321,
+      "step": 264
+    },
+    {
+      "epoch": 0.3298070939639079,
+      "grad_norm": 2.560047149658203,
+      "learning_rate": 4.979243542435424e-06,
+      "loss": 1.7508,
+      "step": 265
+    },
+    {
+      "epoch": 0.3310516490354698,
+      "grad_norm": 3.26595401763916,
+      "learning_rate": 4.976937269372694e-06,
+      "loss": 1.7248,
+      "step": 266
+    },
+    {
+      "epoch": 0.33229620410703176,
+      "grad_norm": 2.929210662841797,
+      "learning_rate": 4.974630996309964e-06,
+      "loss": 1.697,
+      "step": 267
+    },
+    {
+      "epoch": 0.33354075917859366,
+      "grad_norm": 3.1526286602020264,
+      "learning_rate": 4.972324723247233e-06,
+      "loss": 1.7385,
+      "step": 268
+    },
+    {
+      "epoch": 0.33478531425015556,
+      "grad_norm": 3.001619577407837,
+      "learning_rate": 4.970018450184502e-06,
+      "loss": 1.7072,
+      "step": 269
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "grad_norm": 3.2464189529418945,
+      "learning_rate": 4.9677121771217715e-06,
+      "loss": 1.7361,
+      "step": 270
+    },
+    {
+      "epoch": 0.33602986932171747,
+      "eval_loss": 1.860226035118103,
+      "eval_runtime": 46.6881,
+      "eval_samples_per_second": 21.419,
+      "eval_steps_per_second": 0.9,
+      "step": 270
+    },
+    {
+      "epoch": 0.3372744243932794,
+      "grad_norm": 3.2104530334472656,
+      "learning_rate": 4.965405904059041e-06,
+      "loss": 1.7171,
+      "step": 271
+    },
+    {
+      "epoch": 0.33851897946484133,
+      "grad_norm": 3.146847724914551,
+      "learning_rate": 4.96309963099631e-06,
+      "loss": 1.7311,
+      "step": 272
+    },
+    {
+      "epoch": 0.33976353453640323,
+      "grad_norm": 3.1431286334991455,
+      "learning_rate": 4.96079335793358e-06,
+      "loss": 1.7345,
+      "step": 273
+    },
+    {
+      "epoch": 0.34100808960796514,
+      "grad_norm": 2.847163677215576,
+      "learning_rate": 4.958487084870849e-06,
+      "loss": 1.7262,
+      "step": 274
+    },
+    {
+      "epoch": 0.3422526446795271,
+      "grad_norm": 3.1987810134887695,
+      "learning_rate": 4.956180811808119e-06,
+      "loss": 1.7945,
+      "step": 275
+    },
+    {
+      "epoch": 0.343497199751089,
+      "grad_norm": 2.7475385665893555,
+      "learning_rate": 4.953874538745388e-06,
+      "loss": 1.76,
+      "step": 276
+    },
+    {
+      "epoch": 0.3447417548226509,
+      "grad_norm": 3.0427663326263428,
+      "learning_rate": 4.9515682656826574e-06,
+      "loss": 1.7324,
+      "step": 277
+    },
+    {
+      "epoch": 0.3459863098942128,
+      "grad_norm": 3.605212688446045,
+      "learning_rate": 4.949261992619927e-06,
+      "loss": 1.7712,
+      "step": 278
+    },
+    {
+      "epoch": 0.3472308649657747,
+      "grad_norm": 3.0564935207366943,
+      "learning_rate": 4.946955719557196e-06,
+      "loss": 1.6792,
+      "step": 279
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "grad_norm": 3.1069741249084473,
+      "learning_rate": 4.944649446494466e-06,
+      "loss": 1.7145,
+      "step": 280
+    },
+    {
+      "epoch": 0.34847542003733667,
+      "eval_loss": 1.8537051677703857,
+      "eval_runtime": 47.4813,
+      "eval_samples_per_second": 21.061,
+      "eval_steps_per_second": 0.885,
+      "step": 280
+    },
+    {
+      "epoch": 0.3497199751088986,
+      "grad_norm": 2.8021512031555176,
+      "learning_rate": 4.942343173431734e-06,
+      "loss": 1.7145,
+      "step": 281
+    },
+    {
+      "epoch": 0.3509645301804605,
+      "grad_norm": 3.5549023151397705,
+      "learning_rate": 4.940036900369004e-06,
+      "loss": 1.7642,
+      "step": 282
+    },
+    {
+      "epoch": 0.3522090852520224,
+      "grad_norm": 2.7648985385894775,
+      "learning_rate": 4.937730627306274e-06,
+      "loss": 1.7255,
+      "step": 283
+    },
+    {
+      "epoch": 0.35345364032358434,
+      "grad_norm": 3.0815863609313965,
+      "learning_rate": 4.9354243542435426e-06,
+      "loss": 1.7055,
+      "step": 284
+    },
+    {
+      "epoch": 0.35469819539514624,
+      "grad_norm": 2.9009227752685547,
+      "learning_rate": 4.933118081180812e-06,
+      "loss": 1.7019,
+      "step": 285
+    },
+    {
+      "epoch": 0.35594275046670815,
+      "grad_norm": 2.618429660797119,
+      "learning_rate": 4.930811808118081e-06,
+      "loss": 1.6318,
+      "step": 286
+    },
+    {
+      "epoch": 0.35718730553827005,
+      "grad_norm": 3.368230104446411,
+      "learning_rate": 4.928505535055351e-06,
+      "loss": 1.7304,
+      "step": 287
+    },
+    {
+      "epoch": 0.358431860609832,
+      "grad_norm": 3.256889820098877,
+      "learning_rate": 4.92619926199262e-06,
+      "loss": 1.7019,
+      "step": 288
+    },
+    {
+      "epoch": 0.3596764156813939,
+      "grad_norm": 2.9366304874420166,
+      "learning_rate": 4.92389298892989e-06,
+      "loss": 1.734,
+      "step": 289
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "grad_norm": 3.0393142700195312,
+      "learning_rate": 4.921586715867159e-06,
+      "loss": 1.7504,
+      "step": 290
+    },
+    {
+      "epoch": 0.3609209707529558,
+      "eval_loss": 1.8359886407852173,
+      "eval_runtime": 49.7622,
+      "eval_samples_per_second": 20.096,
+      "eval_steps_per_second": 0.844,
+      "step": 290
+    },
+    {
+      "epoch": 0.3621655258245177,
+      "grad_norm": 2.7288589477539062,
+      "learning_rate": 4.9192804428044285e-06,
+      "loss": 1.7004,
+      "step": 291
+    },
+    {
+      "epoch": 0.3634100808960797,
+      "grad_norm": 3.2793378829956055,
+      "learning_rate": 4.916974169741698e-06,
+      "loss": 1.7028,
+      "step": 292
+    },
+    {
+      "epoch": 0.3646546359676416,
+      "grad_norm": 2.960880756378174,
+      "learning_rate": 4.914667896678967e-06,
+      "loss": 1.6759,
+      "step": 293
+    },
+    {
+      "epoch": 0.3658991910392035,
+      "grad_norm": 2.836421012878418,
+      "learning_rate": 4.912361623616237e-06,
+      "loss": 1.6475,
+      "step": 294
+    },
+    {
+      "epoch": 0.3671437461107654,
+      "grad_norm": 2.9495465755462646,
+      "learning_rate": 4.910055350553506e-06,
+      "loss": 1.7076,
+      "step": 295
+    },
+    {
+      "epoch": 0.3683883011823273,
+      "grad_norm": 2.954730272293091,
+      "learning_rate": 4.907749077490776e-06,
+      "loss": 1.7261,
+      "step": 296
+    },
+    {
+      "epoch": 0.36963285625388925,
+      "grad_norm": 2.8167543411254883,
+      "learning_rate": 4.905442804428044e-06,
+      "loss": 1.6939,
+      "step": 297
+    },
+    {
+      "epoch": 0.37087741132545116,
+      "grad_norm": 2.857316017150879,
+      "learning_rate": 4.903136531365314e-06,
+      "loss": 1.7362,
+      "step": 298
+    },
+    {
+      "epoch": 0.37212196639701306,
+      "grad_norm": 2.8612918853759766,
+      "learning_rate": 4.900830258302584e-06,
+      "loss": 1.6489,
+      "step": 299
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "grad_norm": 2.9618914127349854,
+      "learning_rate": 4.898523985239853e-06,
+      "loss": 1.718,
+      "step": 300
+    },
+    {
+      "epoch": 0.37336652146857496,
+      "eval_loss": 1.827571988105774,
+      "eval_runtime": 46.4552,
+      "eval_samples_per_second": 21.526,
+      "eval_steps_per_second": 0.904,
+      "step": 300
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2409,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.35192822808576e+16,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..6cdf93817e8f6d23b60d79888b226cc66ed2e88b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50e4918f3c49370326e14a7ed2f73b48b05e521f4b3399b53438fac6abee926
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-300/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..163fd475f6ed19c9372434210929f632272db873
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-400/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:552bc326efb6531558fb698944765459d3079943e836ca49f69f744359d536de
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fb28500a6d4d87c1d5a189ff54002e2016c059b6
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-600/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f696fd768ec7cdf318fa0be24333069e1e84831059aad715787e60e916b555ea
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-800/model-00002-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-800/model-00002-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6d20fc2be0be600932fae98abf42b021daa63755
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_hop_tokens4_10M_seed0/runs/checkpoint-800/model-00002-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afc557fbcefc3a97020feab2b75416c762bdfedfa4f8d77ad44991a415d46cd7
+size 2247734920
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..87e0c242806933a1097d38a5f95128c647d430fa
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1000/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5e853ca62578f4d2d2e696ecfbf1360aaa2022e909a07bd83eaa20a7a534b4d
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1400/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1400/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..521c1bb30c31a46352e34f0a7f5b3b2d80045268
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1400/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0d726040e9a2db204b35eb441d472702e1ef6893c8036c19cf3a9c5fe8cef55
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..53ceca1f196d1aacb6e3f5ae89d60c20c751b6f2
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1800/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f5da9fadad6349fdbf23fd5a4999d234f6f5ece9401c645e4c5a8f748929955
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1900/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1900/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e62290323c6bf97e44abf1c18ebb63d3eb1fef34
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-1900/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b957b1e58acbdb2f96f37f0b7491e81b29e02edf94c643ed5bc9cb51893e677
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3e655db6c2f48053aeff52ba2af07ad06e177798
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2000/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdbc3d4a1b3bd2536a423d2621eeff059a74dc0520ba4c3918ef8b54727ead40
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8190e57bebe2d1f7896df3cdf42b08591fb62f32
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2100/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef12395e1e22bdacfbadb969c76e7d20be931215409ad0305f09d6754d273771
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2700/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2700/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0e65805d081f26bb5a16829b47cf6613b0e56116
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2700/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ddcce72931ba4376abfa71716cae279841fda46ae54de01f5d57924130587a4
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e1ba15c668859ea95e4fe629622d66100d2c4e40
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2800/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f41736ce81e446a1c1db9be238a0958e90339e9fdfba6d1acf4d5d10e043d665
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2919/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2919/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7856bf5fa202b43e120173ce7928f2108ed79927
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-2919/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af46b7897e2ba77c00b2bd97eb12f776e89c14565e5b7387ba1977cbecd73ec5
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-400/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-400/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7d60f4ab58e9d16098e40ab05ba31be8b6131931
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-400/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40c8a88cc476336d0e128bf4104b00d4aaa1a76474b583dfc2e5ae06f924d956
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4d0bad81bc49ffd72a6e203f871dfee31ef41417
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-500/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ad64e4a27ea03007e4c4878553ae6a96e3844db225febb3cdc6bbe4fe604cb8
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1ee3731f16d9e5d6aa41d1c4ab4d48d51a906133
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-700/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfdef073ce83bc0917e9b91ee3d61bf9a9d1f954a0aa6e54a946200c20d680bb
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-800/model-00001-of-00002.safetensors b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-800/model-00001-of-00002.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6ef3ed3a93317415a3d9ad4d80fa96cbddd40928
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_reverse_partial_10M_seed0/runs/checkpoint-800/model-00001-of-00002.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a2e8d7328e8216f5e3d4e0fff3443eab18c02682c362bc74992270d9c79868e
+size 4965798912
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/latest b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/latest
new file mode 100644
index 0000000000000000000000000000000000000000..744ae7dbad571b6f37ec6c7066549494261bb59e
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/latest
@@ -0,0 +1 @@
+global_step100
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..44995716d9133cdde1b0eafab9b2cbbb3010cad2
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01b934e6ec558bac6fe169663502cde3d438344e9336e1e7d8afe651e9135428
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..ab8016bddf781c8ef313d7126b4b87d8b0c76b56
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc80f18f7e62b82a61f483f87081652089fe5a8da2188f74fe6f789b8c74274
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..70cef6545138ddc2cc4b0c0ab18344cceb230f38
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c577a7e7176d95ccd3d6ec57d9ef0875615f3974af670a48262ddf57ea767d
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c03498f12c7817d4bbb55eae3a0bc6d0397b6056
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ea5e6938d34d22f9628dfc460e37fb652939f5bb51424123150e7dfaaab414
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..309789d5ccd06f418fbcfb7472a6a9affda68101
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d422bc7d90212ffb8070a743cdbe8ff2dfc5634b1fce8f2e3fa671ee441989eb
+size 17210374
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5684b14e2756b56aa397c1324e239b442162b34a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/tokenizer_config.json
@@ -0,0 +1,2070 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..add60f922ca1aafe3319bee133bc97c17e8e2a8a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/trainer_state.json
@@ -0,0 +1,813 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1095890410958904,
+  "eval_steps": 10,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001095890410958904,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0636,
+      "step": 1
+    },
+    {
+      "epoch": 0.002191780821917808,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0501,
+      "step": 2
+    },
+    {
+      "epoch": 0.0032876712328767125,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0238,
+      "step": 3
+    },
+    {
+      "epoch": 0.004383561643835616,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0346,
+      "step": 4
+    },
+    {
+      "epoch": 0.005479452054794521,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0108,
+      "step": 5
+    },
+    {
+      "epoch": 0.006575342465753425,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.131,
+      "step": 6
+    },
+    {
+      "epoch": 0.007671232876712329,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0488,
+      "step": 7
+    },
+    {
+      "epoch": 0.008767123287671232,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0905,
+      "step": 8
+    },
+    {
+      "epoch": 0.009863013698630137,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0493,
+      "step": 9
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.1038,
+      "step": 10
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "eval_loss": 3.100078821182251,
+      "eval_runtime": 37.5838,
+      "eval_samples_per_second": 26.607,
+      "eval_steps_per_second": 1.118,
+      "step": 10
+    },
+    {
+      "epoch": 0.012054794520547946,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0804,
+      "step": 11
+    },
+    {
+      "epoch": 0.01315068493150685,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0053,
+      "step": 12
+    },
+    {
+      "epoch": 0.014246575342465753,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0607,
+      "step": 13
+    },
+    {
+      "epoch": 0.015342465753424657,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0539,
+      "step": 14
+    },
+    {
+      "epoch": 0.01643835616438356,
+      "grad_norm": 5.420984745025635,
+      "learning_rate": 1.8248175182481753e-08,
+      "loss": 3.0941,
+      "step": 15
+    },
+    {
+      "epoch": 0.017534246575342465,
+      "grad_norm": 5.76840877532959,
+      "learning_rate": 3.6496350364963505e-08,
+      "loss": 3.036,
+      "step": 16
+    },
+    {
+      "epoch": 0.01863013698630137,
+      "grad_norm": 5.905357360839844,
+      "learning_rate": 5.4744525547445264e-08,
+      "loss": 3.0491,
+      "step": 17
+    },
+    {
+      "epoch": 0.019726027397260273,
+      "grad_norm": 5.981553554534912,
+      "learning_rate": 7.299270072992701e-08,
+      "loss": 3.0775,
+      "step": 18
+    },
+    {
+      "epoch": 0.020821917808219178,
+      "grad_norm": 6.240172863006592,
+      "learning_rate": 9.124087591240876e-08,
+      "loss": 3.0345,
+      "step": 19
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "grad_norm": 5.59375,
+      "learning_rate": 1.0948905109489053e-07,
+      "loss": 3.1063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "eval_loss": 3.098820686340332,
+      "eval_runtime": 37.2695,
+      "eval_samples_per_second": 26.832,
+      "eval_steps_per_second": 1.127,
+      "step": 20
+    },
+    {
+      "epoch": 0.023013698630136987,
+      "grad_norm": 5.819795608520508,
+      "learning_rate": 1.2773722627737227e-07,
+      "loss": 3.0808,
+      "step": 21
+    },
+    {
+      "epoch": 0.02410958904109589,
+      "grad_norm": 5.7714762687683105,
+      "learning_rate": 1.4598540145985402e-07,
+      "loss": 3.0055,
+      "step": 22
+    },
+    {
+      "epoch": 0.025205479452054796,
+      "grad_norm": 5.51552152633667,
+      "learning_rate": 1.642335766423358e-07,
+      "loss": 3.0396,
+      "step": 23
+    },
+    {
+      "epoch": 0.0263013698630137,
+      "grad_norm": 5.594012260437012,
+      "learning_rate": 1.8248175182481753e-07,
+      "loss": 3.0824,
+      "step": 24
+    },
+    {
+      "epoch": 0.0273972602739726,
+      "grad_norm": 5.1974029541015625,
+      "learning_rate": 2.0072992700729928e-07,
+      "loss": 3.08,
+      "step": 25
+    },
+    {
+      "epoch": 0.028493150684931506,
+      "grad_norm": 4.733968257904053,
+      "learning_rate": 2.1897810218978106e-07,
+      "loss": 3.0597,
+      "step": 26
+    },
+    {
+      "epoch": 0.02958904109589041,
+      "grad_norm": 4.717319011688232,
+      "learning_rate": 2.3722627737226278e-07,
+      "loss": 3.0696,
+      "step": 27
+    },
+    {
+      "epoch": 0.030684931506849315,
+      "grad_norm": 4.6303534507751465,
+      "learning_rate": 2.5547445255474454e-07,
+      "loss": 3.0273,
+      "step": 28
+    },
+    {
+      "epoch": 0.03178082191780822,
+      "grad_norm": 4.413999557495117,
+      "learning_rate": 2.737226277372263e-07,
+      "loss": 3.0202,
+      "step": 29
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "grad_norm": 4.516043186187744,
+      "learning_rate": 2.9197080291970804e-07,
+      "loss": 2.9926,
+      "step": 30
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "eval_loss": 3.0561797618865967,
+      "eval_runtime": 37.0351,
+      "eval_samples_per_second": 27.001,
+      "eval_steps_per_second": 1.134,
+      "step": 30
+    },
+    {
+      "epoch": 0.03397260273972603,
+      "grad_norm": 3.7179207801818848,
+      "learning_rate": 3.102189781021898e-07,
+      "loss": 3.0086,
+      "step": 31
+    },
+    {
+      "epoch": 0.03506849315068493,
+      "grad_norm": 3.7840425968170166,
+      "learning_rate": 3.284671532846716e-07,
+      "loss": 2.9986,
+      "step": 32
+    },
+    {
+      "epoch": 0.03616438356164384,
+      "grad_norm": 3.843116521835327,
+      "learning_rate": 3.467153284671533e-07,
+      "loss": 2.9677,
+      "step": 33
+    },
+    {
+      "epoch": 0.03726027397260274,
+      "grad_norm": 3.8528714179992676,
+      "learning_rate": 3.6496350364963505e-07,
+      "loss": 2.9732,
+      "step": 34
+    },
+    {
+      "epoch": 0.038356164383561646,
+      "grad_norm": 3.76963210105896,
+      "learning_rate": 3.8321167883211686e-07,
+      "loss": 2.9855,
+      "step": 35
+    },
+    {
+      "epoch": 0.03945205479452055,
+      "grad_norm": 3.6240015029907227,
+      "learning_rate": 4.0145985401459856e-07,
+      "loss": 3.0117,
+      "step": 36
+    },
+    {
+      "epoch": 0.040547945205479455,
+      "grad_norm": 3.1380624771118164,
+      "learning_rate": 4.197080291970803e-07,
+      "loss": 2.9481,
+      "step": 37
+    },
+    {
+      "epoch": 0.041643835616438356,
+      "grad_norm": 3.565329074859619,
+      "learning_rate": 4.379562043795621e-07,
+      "loss": 2.9175,
+      "step": 38
+    },
+    {
+      "epoch": 0.042739726027397264,
+      "grad_norm": 3.8062448501586914,
+      "learning_rate": 4.562043795620438e-07,
+      "loss": 3.0074,
+      "step": 39
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "grad_norm": 3.412046432495117,
+      "learning_rate": 4.7445255474452557e-07,
+      "loss": 2.9119,
+      "step": 40
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "eval_loss": 2.9874370098114014,
+      "eval_runtime": 37.0514,
+      "eval_samples_per_second": 26.99,
+      "eval_steps_per_second": 1.134,
+      "step": 40
+    },
+    {
+      "epoch": 0.044931506849315066,
+      "grad_norm": 3.227437973022461,
+      "learning_rate": 4.927007299270073e-07,
+      "loss": 2.9436,
+      "step": 41
+    },
+    {
+      "epoch": 0.046027397260273974,
+      "grad_norm": 2.9633443355560303,
+      "learning_rate": 5.109489051094891e-07,
+      "loss": 2.967,
+      "step": 42
+    },
+    {
+      "epoch": 0.047123287671232875,
+      "grad_norm": 2.942678928375244,
+      "learning_rate": 5.291970802919708e-07,
+      "loss": 2.9375,
+      "step": 43
+    },
+    {
+      "epoch": 0.04821917808219178,
+      "grad_norm": 2.9806888103485107,
+      "learning_rate": 5.474452554744526e-07,
+      "loss": 2.9474,
+      "step": 44
+    },
+    {
+      "epoch": 0.049315068493150684,
+      "grad_norm": 3.000063419342041,
+      "learning_rate": 5.656934306569343e-07,
+      "loss": 2.9289,
+      "step": 45
+    },
+    {
+      "epoch": 0.05041095890410959,
+      "grad_norm": 3.1481194496154785,
+      "learning_rate": 5.839416058394161e-07,
+      "loss": 2.8869,
+      "step": 46
+    },
+    {
+      "epoch": 0.05150684931506849,
+      "grad_norm": 2.949202060699463,
+      "learning_rate": 6.021897810218979e-07,
+      "loss": 2.8783,
+      "step": 47
+    },
+    {
+      "epoch": 0.0526027397260274,
+      "grad_norm": 2.706526279449463,
+      "learning_rate": 6.204379562043796e-07,
+      "loss": 2.9024,
+      "step": 48
+    },
+    {
+      "epoch": 0.0536986301369863,
+      "grad_norm": 2.7317655086517334,
+      "learning_rate": 6.386861313868614e-07,
+      "loss": 2.8325,
+      "step": 49
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "grad_norm": 2.25644588470459,
+      "learning_rate": 6.569343065693432e-07,
+      "loss": 2.9409,
+      "step": 50
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "eval_loss": 2.909320116043091,
+      "eval_runtime": 37.0714,
+      "eval_samples_per_second": 26.975,
+      "eval_steps_per_second": 1.133,
+      "step": 50
+    },
+    {
+      "epoch": 0.05589041095890411,
+      "grad_norm": 2.8789613246917725,
+      "learning_rate": 6.751824817518248e-07,
+      "loss": 2.8459,
+      "step": 51
+    },
+    {
+      "epoch": 0.05698630136986301,
+      "grad_norm": 2.686873435974121,
+      "learning_rate": 6.934306569343066e-07,
+      "loss": 2.8653,
+      "step": 52
+    },
+    {
+      "epoch": 0.05808219178082192,
+      "grad_norm": 2.54382586479187,
+      "learning_rate": 7.116788321167883e-07,
+      "loss": 2.7973,
+      "step": 53
+    },
+    {
+      "epoch": 0.05917808219178082,
+      "grad_norm": 2.1871724128723145,
+      "learning_rate": 7.299270072992701e-07,
+      "loss": 2.7982,
+      "step": 54
+    },
+    {
+      "epoch": 0.06027397260273973,
+      "grad_norm": 2.145303249359131,
+      "learning_rate": 7.48175182481752e-07,
+      "loss": 2.8195,
+      "step": 55
+    },
+    {
+      "epoch": 0.06136986301369863,
+      "grad_norm": 2.1635069847106934,
+      "learning_rate": 7.664233576642337e-07,
+      "loss": 2.825,
+      "step": 56
+    },
+    {
+      "epoch": 0.06246575342465754,
+      "grad_norm": 2.3530752658843994,
+      "learning_rate": 7.846715328467154e-07,
+      "loss": 2.8407,
+      "step": 57
+    },
+    {
+      "epoch": 0.06356164383561644,
+      "grad_norm": 2.2455942630767822,
+      "learning_rate": 8.029197080291971e-07,
+      "loss": 2.8182,
+      "step": 58
+    },
+    {
+      "epoch": 0.06465753424657535,
+      "grad_norm": 1.8985673189163208,
+      "learning_rate": 8.211678832116789e-07,
+      "loss": 2.772,
+      "step": 59
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "grad_norm": 1.9699499607086182,
+      "learning_rate": 8.394160583941606e-07,
+      "loss": 2.7367,
+      "step": 60
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "eval_loss": 2.8451859951019287,
+      "eval_runtime": 37.0793,
+      "eval_samples_per_second": 26.969,
+      "eval_steps_per_second": 1.133,
+      "step": 60
+    },
+    {
+      "epoch": 0.06684931506849315,
+      "grad_norm": 2.159547805786133,
+      "learning_rate": 8.576642335766425e-07,
+      "loss": 2.7635,
+      "step": 61
+    },
+    {
+      "epoch": 0.06794520547945206,
+      "grad_norm": 2.12705659866333,
+      "learning_rate": 8.759124087591242e-07,
+      "loss": 2.7624,
+      "step": 62
+    },
+    {
+      "epoch": 0.06904109589041096,
+      "grad_norm": 1.8393241167068481,
+      "learning_rate": 8.94160583941606e-07,
+      "loss": 2.7763,
+      "step": 63
+    },
+    {
+      "epoch": 0.07013698630136986,
+      "grad_norm": 1.683522343635559,
+      "learning_rate": 9.124087591240876e-07,
+      "loss": 2.7762,
+      "step": 64
+    },
+    {
+      "epoch": 0.07123287671232877,
+      "grad_norm": 1.9260376691818237,
+      "learning_rate": 9.306569343065694e-07,
+      "loss": 2.7418,
+      "step": 65
+    },
+    {
+      "epoch": 0.07232876712328767,
+      "grad_norm": 2.09322190284729,
+      "learning_rate": 9.489051094890511e-07,
+      "loss": 2.7625,
+      "step": 66
+    },
+    {
+      "epoch": 0.07342465753424658,
+      "grad_norm": 1.9183429479599,
+      "learning_rate": 9.67153284671533e-07,
+      "loss": 2.7453,
+      "step": 67
+    },
+    {
+      "epoch": 0.07452054794520548,
+      "grad_norm": 1.9792778491973877,
+      "learning_rate": 9.854014598540146e-07,
+      "loss": 2.7858,
+      "step": 68
+    },
+    {
+      "epoch": 0.07561643835616438,
+      "grad_norm": 2.1397311687469482,
+      "learning_rate": 1.0036496350364965e-06,
+      "loss": 2.7242,
+      "step": 69
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "grad_norm": 1.8556463718414307,
+      "learning_rate": 1.0218978102189781e-06,
+      "loss": 2.6853,
+      "step": 70
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "eval_loss": 2.7787046432495117,
+      "eval_runtime": 37.1017,
+      "eval_samples_per_second": 26.953,
+      "eval_steps_per_second": 1.132,
+      "step": 70
+    },
+    {
+      "epoch": 0.07780821917808219,
+      "grad_norm": 1.9776800870895386,
+      "learning_rate": 1.04014598540146e-06,
+      "loss": 2.749,
+      "step": 71
+    },
+    {
+      "epoch": 0.0789041095890411,
+      "grad_norm": 2.3972630500793457,
+      "learning_rate": 1.0583941605839416e-06,
+      "loss": 2.6457,
+      "step": 72
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.090210199356079,
+      "learning_rate": 1.0766423357664233e-06,
+      "loss": 2.6401,
+      "step": 73
+    },
+    {
+      "epoch": 0.08109589041095891,
+      "grad_norm": 2.4117701053619385,
+      "learning_rate": 1.0948905109489052e-06,
+      "loss": 2.6649,
+      "step": 74
+    },
+    {
+      "epoch": 0.0821917808219178,
+      "grad_norm": 2.4845173358917236,
+      "learning_rate": 1.113138686131387e-06,
+      "loss": 2.6855,
+      "step": 75
+    },
+    {
+      "epoch": 0.08328767123287671,
+      "grad_norm": 1.7779242992401123,
+      "learning_rate": 1.1313868613138687e-06,
+      "loss": 2.6926,
+      "step": 76
+    },
+    {
+      "epoch": 0.08438356164383562,
+      "grad_norm": 2.3163535594940186,
+      "learning_rate": 1.1496350364963505e-06,
+      "loss": 2.6302,
+      "step": 77
+    },
+    {
+      "epoch": 0.08547945205479453,
+      "grad_norm": 2.2296528816223145,
+      "learning_rate": 1.1678832116788322e-06,
+      "loss": 2.6054,
+      "step": 78
+    },
+    {
+      "epoch": 0.08657534246575342,
+      "grad_norm": 2.253431797027588,
+      "learning_rate": 1.1861313868613138e-06,
+      "loss": 2.6104,
+      "step": 79
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "grad_norm": 2.490609645843506,
+      "learning_rate": 1.2043795620437959e-06,
+      "loss": 2.6078,
+      "step": 80
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "eval_loss": 2.706103563308716,
+      "eval_runtime": 37.0892,
+      "eval_samples_per_second": 26.962,
+      "eval_steps_per_second": 1.132,
+      "step": 80
+    },
+    {
+      "epoch": 0.08876712328767124,
+      "grad_norm": 2.3048248291015625,
+      "learning_rate": 1.2226277372262775e-06,
+      "loss": 2.6096,
+      "step": 81
+    },
+    {
+      "epoch": 0.08986301369863013,
+      "grad_norm": 2.440666913986206,
+      "learning_rate": 1.2408759124087592e-06,
+      "loss": 2.6412,
+      "step": 82
+    },
+    {
+      "epoch": 0.09095890410958904,
+      "grad_norm": 1.9456374645233154,
+      "learning_rate": 1.259124087591241e-06,
+      "loss": 2.591,
+      "step": 83
+    },
+    {
+      "epoch": 0.09205479452054795,
+      "grad_norm": 2.598172664642334,
+      "learning_rate": 1.2773722627737229e-06,
+      "loss": 2.6093,
+      "step": 84
+    },
+    {
+      "epoch": 0.09315068493150686,
+      "grad_norm": 2.2587032318115234,
+      "learning_rate": 1.2956204379562045e-06,
+      "loss": 2.5949,
+      "step": 85
+    },
+    {
+      "epoch": 0.09424657534246575,
+      "grad_norm": 2.2927448749542236,
+      "learning_rate": 1.3138686131386864e-06,
+      "loss": 2.5464,
+      "step": 86
+    },
+    {
+      "epoch": 0.09534246575342466,
+      "grad_norm": 2.5940468311309814,
+      "learning_rate": 1.332116788321168e-06,
+      "loss": 2.5613,
+      "step": 87
+    },
+    {
+      "epoch": 0.09643835616438357,
+      "grad_norm": 2.073207378387451,
+      "learning_rate": 1.3503649635036497e-06,
+      "loss": 2.4955,
+      "step": 88
+    },
+    {
+      "epoch": 0.09753424657534246,
+      "grad_norm": 3.211766242980957,
+      "learning_rate": 1.3686131386861315e-06,
+      "loss": 2.5315,
+      "step": 89
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "grad_norm": 2.2590038776397705,
+      "learning_rate": 1.3868613138686132e-06,
+      "loss": 2.4641,
+      "step": 90
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "eval_loss": 2.625087261199951,
+      "eval_runtime": 37.0852,
+      "eval_samples_per_second": 26.965,
+      "eval_steps_per_second": 1.133,
+      "step": 90
+    },
+    {
+      "epoch": 0.09972602739726028,
+      "grad_norm": 2.3959836959838867,
+      "learning_rate": 1.405109489051095e-06,
+      "loss": 2.5537,
+      "step": 91
+    },
+    {
+      "epoch": 0.10082191780821918,
+      "grad_norm": 2.596761465072632,
+      "learning_rate": 1.4233576642335767e-06,
+      "loss": 2.5135,
+      "step": 92
+    },
+    {
+      "epoch": 0.10191780821917808,
+      "grad_norm": 2.475759267807007,
+      "learning_rate": 1.4416058394160583e-06,
+      "loss": 2.5436,
+      "step": 93
+    },
+    {
+      "epoch": 0.10301369863013699,
+      "grad_norm": 3.0613298416137695,
+      "learning_rate": 1.4598540145985402e-06,
+      "loss": 2.5066,
+      "step": 94
+    },
+    {
+      "epoch": 0.10410958904109589,
+      "grad_norm": 2.406097650527954,
+      "learning_rate": 1.478102189781022e-06,
+      "loss": 2.4646,
+      "step": 95
+    },
+    {
+      "epoch": 0.1052054794520548,
+      "grad_norm": 2.5872504711151123,
+      "learning_rate": 1.496350364963504e-06,
+      "loss": 2.5351,
+      "step": 96
+    },
+    {
+      "epoch": 0.1063013698630137,
+      "grad_norm": 2.4294471740722656,
+      "learning_rate": 1.5145985401459856e-06,
+      "loss": 2.4953,
+      "step": 97
+    },
+    {
+      "epoch": 0.1073972602739726,
+      "grad_norm": 2.674884080886841,
+      "learning_rate": 1.5328467153284674e-06,
+      "loss": 2.5133,
+      "step": 98
+    },
+    {
+      "epoch": 0.10849315068493151,
+      "grad_norm": 2.5096852779388428,
+      "learning_rate": 1.551094890510949e-06,
+      "loss": 2.4693,
+      "step": 99
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "grad_norm": 2.7390334606170654,
+      "learning_rate": 1.5693430656934307e-06,
+      "loss": 2.4195,
+      "step": 100
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "eval_loss": 2.55997371673584,
+      "eval_runtime": 37.0732,
+      "eval_samples_per_second": 26.974,
+      "eval_steps_per_second": 1.133,
+      "step": 100
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2736,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.11730940936192e+16,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b7adbe5a2dbf27f05befec6cc65fbc93e90eefd0
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa882279512a8c278376c0c601206933f3c5662ddcd3c13c53bf32ae3680d14
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-100/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/latest b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/latest
new file mode 100644
index 0000000000000000000000000000000000000000..cb24338af451be6d3ccddc18a950b58a70589761
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/latest
@@ -0,0 +1 @@
+global_step1700
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..173f12bde4ca76ddf4b03b0964b7524eec1d0cf8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103551ff1c3e292b45450c5a23cdaf1958c78f242d8cf8b564df9603b5e96d5c
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b0aed44f2fa8fd35795e9b8bfde272b7a98d0ec9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d69b787213a6aea61f516b084db235c8c80de8bef9f33a5157ae51d85fa9135
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..487463491926bb630fa016cfe776df854285e817
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffe85d84d1459bc77a8988c2ddaa541ec71b9112c567063ec91af255e984793b
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e2599b24bc77b7cbf5b941e14863dc395eb0aee
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d06926a97f807ed50063a0fbaaef06f637672c0bf846d322b5f2fd5d678dc96
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..309789d5ccd06f418fbcfb7472a6a9affda68101
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d422bc7d90212ffb8070a743cdbe8ff2dfc5634b1fce8f2e3fa671ee441989eb
+size 17210374
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5684b14e2756b56aa397c1324e239b442162b34a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/tokenizer_config.json
@@ -0,0 +1,2070 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6172355a8998ff0c8298c96f932c565c4efd31c
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/trainer_state.json
@@ -0,0 +1,13293 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.8630136986301369,
+  "eval_steps": 10,
+  "global_step": 1700,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001095890410958904,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0636,
+      "step": 1
+    },
+    {
+      "epoch": 0.002191780821917808,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0501,
+      "step": 2
+    },
+    {
+      "epoch": 0.0032876712328767125,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0238,
+      "step": 3
+    },
+    {
+      "epoch": 0.004383561643835616,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0346,
+      "step": 4
+    },
+    {
+      "epoch": 0.005479452054794521,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0108,
+      "step": 5
+    },
+    {
+      "epoch": 0.006575342465753425,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.131,
+      "step": 6
+    },
+    {
+      "epoch": 0.007671232876712329,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0488,
+      "step": 7
+    },
+    {
+      "epoch": 0.008767123287671232,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0905,
+      "step": 8
+    },
+    {
+      "epoch": 0.009863013698630137,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0493,
+      "step": 9
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.1038,
+      "step": 10
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "eval_loss": 3.100078821182251,
+      "eval_runtime": 37.5838,
+      "eval_samples_per_second": 26.607,
+      "eval_steps_per_second": 1.118,
+      "step": 10
+    },
+    {
+      "epoch": 0.012054794520547946,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0804,
+      "step": 11
+    },
+    {
+      "epoch": 0.01315068493150685,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0053,
+      "step": 12
+    },
+    {
+      "epoch": 0.014246575342465753,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0607,
+      "step": 13
+    },
+    {
+      "epoch": 0.015342465753424657,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0539,
+      "step": 14
+    },
+    {
+      "epoch": 0.01643835616438356,
+      "grad_norm": 5.420984745025635,
+      "learning_rate": 1.8248175182481753e-08,
+      "loss": 3.0941,
+      "step": 15
+    },
+    {
+      "epoch": 0.017534246575342465,
+      "grad_norm": 5.76840877532959,
+      "learning_rate": 3.6496350364963505e-08,
+      "loss": 3.036,
+      "step": 16
+    },
+    {
+      "epoch": 0.01863013698630137,
+      "grad_norm": 5.905357360839844,
+      "learning_rate": 5.4744525547445264e-08,
+      "loss": 3.0491,
+      "step": 17
+    },
+    {
+      "epoch": 0.019726027397260273,
+      "grad_norm": 5.981553554534912,
+      "learning_rate": 7.299270072992701e-08,
+      "loss": 3.0775,
+      "step": 18
+    },
+    {
+      "epoch": 0.020821917808219178,
+      "grad_norm": 6.240172863006592,
+      "learning_rate": 9.124087591240876e-08,
+      "loss": 3.0345,
+      "step": 19
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "grad_norm": 5.59375,
+      "learning_rate": 1.0948905109489053e-07,
+      "loss": 3.1063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "eval_loss": 3.098820686340332,
+      "eval_runtime": 37.2695,
+      "eval_samples_per_second": 26.832,
+      "eval_steps_per_second": 1.127,
+      "step": 20
+    },
+    {
+      "epoch": 0.023013698630136987,
+      "grad_norm": 5.819795608520508,
+      "learning_rate": 1.2773722627737227e-07,
+      "loss": 3.0808,
+      "step": 21
+    },
+    {
+      "epoch": 0.02410958904109589,
+      "grad_norm": 5.7714762687683105,
+      "learning_rate": 1.4598540145985402e-07,
+      "loss": 3.0055,
+      "step": 22
+    },
+    {
+      "epoch": 0.025205479452054796,
+      "grad_norm": 5.51552152633667,
+      "learning_rate": 1.642335766423358e-07,
+      "loss": 3.0396,
+      "step": 23
+    },
+    {
+      "epoch": 0.0263013698630137,
+      "grad_norm": 5.594012260437012,
+      "learning_rate": 1.8248175182481753e-07,
+      "loss": 3.0824,
+      "step": 24
+    },
+    {
+      "epoch": 0.0273972602739726,
+      "grad_norm": 5.1974029541015625,
+      "learning_rate": 2.0072992700729928e-07,
+      "loss": 3.08,
+      "step": 25
+    },
+    {
+      "epoch": 0.028493150684931506,
+      "grad_norm": 4.733968257904053,
+      "learning_rate": 2.1897810218978106e-07,
+      "loss": 3.0597,
+      "step": 26
+    },
+    {
+      "epoch": 0.02958904109589041,
+      "grad_norm": 4.717319011688232,
+      "learning_rate": 2.3722627737226278e-07,
+      "loss": 3.0696,
+      "step": 27
+    },
+    {
+      "epoch": 0.030684931506849315,
+      "grad_norm": 4.6303534507751465,
+      "learning_rate": 2.5547445255474454e-07,
+      "loss": 3.0273,
+      "step": 28
+    },
+    {
+      "epoch": 0.03178082191780822,
+      "grad_norm": 4.413999557495117,
+      "learning_rate": 2.737226277372263e-07,
+      "loss": 3.0202,
+      "step": 29
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "grad_norm": 4.516043186187744,
+      "learning_rate": 2.9197080291970804e-07,
+      "loss": 2.9926,
+      "step": 30
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "eval_loss": 3.0561797618865967,
+      "eval_runtime": 37.0351,
+      "eval_samples_per_second": 27.001,
+      "eval_steps_per_second": 1.134,
+      "step": 30
+    },
+    {
+      "epoch": 0.03397260273972603,
+      "grad_norm": 3.7179207801818848,
+      "learning_rate": 3.102189781021898e-07,
+      "loss": 3.0086,
+      "step": 31
+    },
+    {
+      "epoch": 0.03506849315068493,
+      "grad_norm": 3.7840425968170166,
+      "learning_rate": 3.284671532846716e-07,
+      "loss": 2.9986,
+      "step": 32
+    },
+    {
+      "epoch": 0.03616438356164384,
+      "grad_norm": 3.843116521835327,
+      "learning_rate": 3.467153284671533e-07,
+      "loss": 2.9677,
+      "step": 33
+    },
+    {
+      "epoch": 0.03726027397260274,
+      "grad_norm": 3.8528714179992676,
+      "learning_rate": 3.6496350364963505e-07,
+      "loss": 2.9732,
+      "step": 34
+    },
+    {
+      "epoch": 0.038356164383561646,
+      "grad_norm": 3.76963210105896,
+      "learning_rate": 3.8321167883211686e-07,
+      "loss": 2.9855,
+      "step": 35
+    },
+    {
+      "epoch": 0.03945205479452055,
+      "grad_norm": 3.6240015029907227,
+      "learning_rate": 4.0145985401459856e-07,
+      "loss": 3.0117,
+      "step": 36
+    },
+    {
+      "epoch": 0.040547945205479455,
+      "grad_norm": 3.1380624771118164,
+      "learning_rate": 4.197080291970803e-07,
+      "loss": 2.9481,
+      "step": 37
+    },
+    {
+      "epoch": 0.041643835616438356,
+      "grad_norm": 3.565329074859619,
+      "learning_rate": 4.379562043795621e-07,
+      "loss": 2.9175,
+      "step": 38
+    },
+    {
+      "epoch": 0.042739726027397264,
+      "grad_norm": 3.8062448501586914,
+      "learning_rate": 4.562043795620438e-07,
+      "loss": 3.0074,
+      "step": 39
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "grad_norm": 3.412046432495117,
+      "learning_rate": 4.7445255474452557e-07,
+      "loss": 2.9119,
+      "step": 40
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "eval_loss": 2.9874370098114014,
+      "eval_runtime": 37.0514,
+      "eval_samples_per_second": 26.99,
+      "eval_steps_per_second": 1.134,
+      "step": 40
+    },
+    {
+      "epoch": 0.044931506849315066,
+      "grad_norm": 3.227437973022461,
+      "learning_rate": 4.927007299270073e-07,
+      "loss": 2.9436,
+      "step": 41
+    },
+    {
+      "epoch": 0.046027397260273974,
+      "grad_norm": 2.9633443355560303,
+      "learning_rate": 5.109489051094891e-07,
+      "loss": 2.967,
+      "step": 42
+    },
+    {
+      "epoch": 0.047123287671232875,
+      "grad_norm": 2.942678928375244,
+      "learning_rate": 5.291970802919708e-07,
+      "loss": 2.9375,
+      "step": 43
+    },
+    {
+      "epoch": 0.04821917808219178,
+      "grad_norm": 2.9806888103485107,
+      "learning_rate": 5.474452554744526e-07,
+      "loss": 2.9474,
+      "step": 44
+    },
+    {
+      "epoch": 0.049315068493150684,
+      "grad_norm": 3.000063419342041,
+      "learning_rate": 5.656934306569343e-07,
+      "loss": 2.9289,
+      "step": 45
+    },
+    {
+      "epoch": 0.05041095890410959,
+      "grad_norm": 3.1481194496154785,
+      "learning_rate": 5.839416058394161e-07,
+      "loss": 2.8869,
+      "step": 46
+    },
+    {
+      "epoch": 0.05150684931506849,
+      "grad_norm": 2.949202060699463,
+      "learning_rate": 6.021897810218979e-07,
+      "loss": 2.8783,
+      "step": 47
+    },
+    {
+      "epoch": 0.0526027397260274,
+      "grad_norm": 2.706526279449463,
+      "learning_rate": 6.204379562043796e-07,
+      "loss": 2.9024,
+      "step": 48
+    },
+    {
+      "epoch": 0.0536986301369863,
+      "grad_norm": 2.7317655086517334,
+      "learning_rate": 6.386861313868614e-07,
+      "loss": 2.8325,
+      "step": 49
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "grad_norm": 2.25644588470459,
+      "learning_rate": 6.569343065693432e-07,
+      "loss": 2.9409,
+      "step": 50
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "eval_loss": 2.909320116043091,
+      "eval_runtime": 37.0714,
+      "eval_samples_per_second": 26.975,
+      "eval_steps_per_second": 1.133,
+      "step": 50
+    },
+    {
+      "epoch": 0.05589041095890411,
+      "grad_norm": 2.8789613246917725,
+      "learning_rate": 6.751824817518248e-07,
+      "loss": 2.8459,
+      "step": 51
+    },
+    {
+      "epoch": 0.05698630136986301,
+      "grad_norm": 2.686873435974121,
+      "learning_rate": 6.934306569343066e-07,
+      "loss": 2.8653,
+      "step": 52
+    },
+    {
+      "epoch": 0.05808219178082192,
+      "grad_norm": 2.54382586479187,
+      "learning_rate": 7.116788321167883e-07,
+      "loss": 2.7973,
+      "step": 53
+    },
+    {
+      "epoch": 0.05917808219178082,
+      "grad_norm": 2.1871724128723145,
+      "learning_rate": 7.299270072992701e-07,
+      "loss": 2.7982,
+      "step": 54
+    },
+    {
+      "epoch": 0.06027397260273973,
+      "grad_norm": 2.145303249359131,
+      "learning_rate": 7.48175182481752e-07,
+      "loss": 2.8195,
+      "step": 55
+    },
+    {
+      "epoch": 0.06136986301369863,
+      "grad_norm": 2.1635069847106934,
+      "learning_rate": 7.664233576642337e-07,
+      "loss": 2.825,
+      "step": 56
+    },
+    {
+      "epoch": 0.06246575342465754,
+      "grad_norm": 2.3530752658843994,
+      "learning_rate": 7.846715328467154e-07,
+      "loss": 2.8407,
+      "step": 57
+    },
+    {
+      "epoch": 0.06356164383561644,
+      "grad_norm": 2.2455942630767822,
+      "learning_rate": 8.029197080291971e-07,
+      "loss": 2.8182,
+      "step": 58
+    },
+    {
+      "epoch": 0.06465753424657535,
+      "grad_norm": 1.8985673189163208,
+      "learning_rate": 8.211678832116789e-07,
+      "loss": 2.772,
+      "step": 59
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "grad_norm": 1.9699499607086182,
+      "learning_rate": 8.394160583941606e-07,
+      "loss": 2.7367,
+      "step": 60
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "eval_loss": 2.8451859951019287,
+      "eval_runtime": 37.0793,
+      "eval_samples_per_second": 26.969,
+      "eval_steps_per_second": 1.133,
+      "step": 60
+    },
+    {
+      "epoch": 0.06684931506849315,
+      "grad_norm": 2.159547805786133,
+      "learning_rate": 8.576642335766425e-07,
+      "loss": 2.7635,
+      "step": 61
+    },
+    {
+      "epoch": 0.06794520547945206,
+      "grad_norm": 2.12705659866333,
+      "learning_rate": 8.759124087591242e-07,
+      "loss": 2.7624,
+      "step": 62
+    },
+    {
+      "epoch": 0.06904109589041096,
+      "grad_norm": 1.8393241167068481,
+      "learning_rate": 8.94160583941606e-07,
+      "loss": 2.7763,
+      "step": 63
+    },
+    {
+      "epoch": 0.07013698630136986,
+      "grad_norm": 1.683522343635559,
+      "learning_rate": 9.124087591240876e-07,
+      "loss": 2.7762,
+      "step": 64
+    },
+    {
+      "epoch": 0.07123287671232877,
+      "grad_norm": 1.9260376691818237,
+      "learning_rate": 9.306569343065694e-07,
+      "loss": 2.7418,
+      "step": 65
+    },
+    {
+      "epoch": 0.07232876712328767,
+      "grad_norm": 2.09322190284729,
+      "learning_rate": 9.489051094890511e-07,
+      "loss": 2.7625,
+      "step": 66
+    },
+    {
+      "epoch": 0.07342465753424658,
+      "grad_norm": 1.9183429479599,
+      "learning_rate": 9.67153284671533e-07,
+      "loss": 2.7453,
+      "step": 67
+    },
+    {
+      "epoch": 0.07452054794520548,
+      "grad_norm": 1.9792778491973877,
+      "learning_rate": 9.854014598540146e-07,
+      "loss": 2.7858,
+      "step": 68
+    },
+    {
+      "epoch": 0.07561643835616438,
+      "grad_norm": 2.1397311687469482,
+      "learning_rate": 1.0036496350364965e-06,
+      "loss": 2.7242,
+      "step": 69
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "grad_norm": 1.8556463718414307,
+      "learning_rate": 1.0218978102189781e-06,
+      "loss": 2.6853,
+      "step": 70
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "eval_loss": 2.7787046432495117,
+      "eval_runtime": 37.1017,
+      "eval_samples_per_second": 26.953,
+      "eval_steps_per_second": 1.132,
+      "step": 70
+    },
+    {
+      "epoch": 0.07780821917808219,
+      "grad_norm": 1.9776800870895386,
+      "learning_rate": 1.04014598540146e-06,
+      "loss": 2.749,
+      "step": 71
+    },
+    {
+      "epoch": 0.0789041095890411,
+      "grad_norm": 2.3972630500793457,
+      "learning_rate": 1.0583941605839416e-06,
+      "loss": 2.6457,
+      "step": 72
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.090210199356079,
+      "learning_rate": 1.0766423357664233e-06,
+      "loss": 2.6401,
+      "step": 73
+    },
+    {
+      "epoch": 0.08109589041095891,
+      "grad_norm": 2.4117701053619385,
+      "learning_rate": 1.0948905109489052e-06,
+      "loss": 2.6649,
+      "step": 74
+    },
+    {
+      "epoch": 0.0821917808219178,
+      "grad_norm": 2.4845173358917236,
+      "learning_rate": 1.113138686131387e-06,
+      "loss": 2.6855,
+      "step": 75
+    },
+    {
+      "epoch": 0.08328767123287671,
+      "grad_norm": 1.7779242992401123,
+      "learning_rate": 1.1313868613138687e-06,
+      "loss": 2.6926,
+      "step": 76
+    },
+    {
+      "epoch": 0.08438356164383562,
+      "grad_norm": 2.3163535594940186,
+      "learning_rate": 1.1496350364963505e-06,
+      "loss": 2.6302,
+      "step": 77
+    },
+    {
+      "epoch": 0.08547945205479453,
+      "grad_norm": 2.2296528816223145,
+      "learning_rate": 1.1678832116788322e-06,
+      "loss": 2.6054,
+      "step": 78
+    },
+    {
+      "epoch": 0.08657534246575342,
+      "grad_norm": 2.253431797027588,
+      "learning_rate": 1.1861313868613138e-06,
+      "loss": 2.6104,
+      "step": 79
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "grad_norm": 2.490609645843506,
+      "learning_rate": 1.2043795620437959e-06,
+      "loss": 2.6078,
+      "step": 80
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "eval_loss": 2.706103563308716,
+      "eval_runtime": 37.0892,
+      "eval_samples_per_second": 26.962,
+      "eval_steps_per_second": 1.132,
+      "step": 80
+    },
+    {
+      "epoch": 0.08876712328767124,
+      "grad_norm": 2.3048248291015625,
+      "learning_rate": 1.2226277372262775e-06,
+      "loss": 2.6096,
+      "step": 81
+    },
+    {
+      "epoch": 0.08986301369863013,
+      "grad_norm": 2.440666913986206,
+      "learning_rate": 1.2408759124087592e-06,
+      "loss": 2.6412,
+      "step": 82
+    },
+    {
+      "epoch": 0.09095890410958904,
+      "grad_norm": 1.9456374645233154,
+      "learning_rate": 1.259124087591241e-06,
+      "loss": 2.591,
+      "step": 83
+    },
+    {
+      "epoch": 0.09205479452054795,
+      "grad_norm": 2.598172664642334,
+      "learning_rate": 1.2773722627737229e-06,
+      "loss": 2.6093,
+      "step": 84
+    },
+    {
+      "epoch": 0.09315068493150686,
+      "grad_norm": 2.2587032318115234,
+      "learning_rate": 1.2956204379562045e-06,
+      "loss": 2.5949,
+      "step": 85
+    },
+    {
+      "epoch": 0.09424657534246575,
+      "grad_norm": 2.2927448749542236,
+      "learning_rate": 1.3138686131386864e-06,
+      "loss": 2.5464,
+      "step": 86
+    },
+    {
+      "epoch": 0.09534246575342466,
+      "grad_norm": 2.5940468311309814,
+      "learning_rate": 1.332116788321168e-06,
+      "loss": 2.5613,
+      "step": 87
+    },
+    {
+      "epoch": 0.09643835616438357,
+      "grad_norm": 2.073207378387451,
+      "learning_rate": 1.3503649635036497e-06,
+      "loss": 2.4955,
+      "step": 88
+    },
+    {
+      "epoch": 0.09753424657534246,
+      "grad_norm": 3.211766242980957,
+      "learning_rate": 1.3686131386861315e-06,
+      "loss": 2.5315,
+      "step": 89
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "grad_norm": 2.2590038776397705,
+      "learning_rate": 1.3868613138686132e-06,
+      "loss": 2.4641,
+      "step": 90
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "eval_loss": 2.625087261199951,
+      "eval_runtime": 37.0852,
+      "eval_samples_per_second": 26.965,
+      "eval_steps_per_second": 1.133,
+      "step": 90
+    },
+    {
+      "epoch": 0.09972602739726028,
+      "grad_norm": 2.3959836959838867,
+      "learning_rate": 1.405109489051095e-06,
+      "loss": 2.5537,
+      "step": 91
+    },
+    {
+      "epoch": 0.10082191780821918,
+      "grad_norm": 2.596761465072632,
+      "learning_rate": 1.4233576642335767e-06,
+      "loss": 2.5135,
+      "step": 92
+    },
+    {
+      "epoch": 0.10191780821917808,
+      "grad_norm": 2.475759267807007,
+      "learning_rate": 1.4416058394160583e-06,
+      "loss": 2.5436,
+      "step": 93
+    },
+    {
+      "epoch": 0.10301369863013699,
+      "grad_norm": 3.0613298416137695,
+      "learning_rate": 1.4598540145985402e-06,
+      "loss": 2.5066,
+      "step": 94
+    },
+    {
+      "epoch": 0.10410958904109589,
+      "grad_norm": 2.406097650527954,
+      "learning_rate": 1.478102189781022e-06,
+      "loss": 2.4646,
+      "step": 95
+    },
+    {
+      "epoch": 0.1052054794520548,
+      "grad_norm": 2.5872504711151123,
+      "learning_rate": 1.496350364963504e-06,
+      "loss": 2.5351,
+      "step": 96
+    },
+    {
+      "epoch": 0.1063013698630137,
+      "grad_norm": 2.4294471740722656,
+      "learning_rate": 1.5145985401459856e-06,
+      "loss": 2.4953,
+      "step": 97
+    },
+    {
+      "epoch": 0.1073972602739726,
+      "grad_norm": 2.674884080886841,
+      "learning_rate": 1.5328467153284674e-06,
+      "loss": 2.5133,
+      "step": 98
+    },
+    {
+      "epoch": 0.10849315068493151,
+      "grad_norm": 2.5096852779388428,
+      "learning_rate": 1.551094890510949e-06,
+      "loss": 2.4693,
+      "step": 99
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "grad_norm": 2.7390334606170654,
+      "learning_rate": 1.5693430656934307e-06,
+      "loss": 2.4195,
+      "step": 100
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "eval_loss": 2.55997371673584,
+      "eval_runtime": 37.0732,
+      "eval_samples_per_second": 26.974,
+      "eval_steps_per_second": 1.133,
+      "step": 100
+    },
+    {
+      "epoch": 0.11068493150684931,
+      "grad_norm": 2.8887312412261963,
+      "learning_rate": 1.5875912408759126e-06,
+      "loss": 2.4193,
+      "step": 101
+    },
+    {
+      "epoch": 0.11178082191780822,
+      "grad_norm": 3.0721421241760254,
+      "learning_rate": 1.6058394160583942e-06,
+      "loss": 2.4709,
+      "step": 102
+    },
+    {
+      "epoch": 0.11287671232876713,
+      "grad_norm": 2.728156566619873,
+      "learning_rate": 1.624087591240876e-06,
+      "loss": 2.5105,
+      "step": 103
+    },
+    {
+      "epoch": 0.11397260273972602,
+      "grad_norm": 3.1214346885681152,
+      "learning_rate": 1.6423357664233577e-06,
+      "loss": 2.4316,
+      "step": 104
+    },
+    {
+      "epoch": 0.11506849315068493,
+      "grad_norm": 2.9249517917633057,
+      "learning_rate": 1.6605839416058394e-06,
+      "loss": 2.3737,
+      "step": 105
+    },
+    {
+      "epoch": 0.11616438356164384,
+      "grad_norm": 2.6646173000335693,
+      "learning_rate": 1.6788321167883212e-06,
+      "loss": 2.4165,
+      "step": 106
+    },
+    {
+      "epoch": 0.11726027397260275,
+      "grad_norm": 3.108067750930786,
+      "learning_rate": 1.6970802919708029e-06,
+      "loss": 2.4217,
+      "step": 107
+    },
+    {
+      "epoch": 0.11835616438356164,
+      "grad_norm": 3.142338991165161,
+      "learning_rate": 1.715328467153285e-06,
+      "loss": 2.3527,
+      "step": 108
+    },
+    {
+      "epoch": 0.11945205479452055,
+      "grad_norm": 2.9078011512756348,
+      "learning_rate": 1.7335766423357666e-06,
+      "loss": 2.3927,
+      "step": 109
+    },
+    {
+      "epoch": 0.12054794520547946,
+      "grad_norm": 3.40766978263855,
+      "learning_rate": 1.7518248175182485e-06,
+      "loss": 2.4021,
+      "step": 110
+    },
+    {
+      "epoch": 0.12054794520547946,
+      "eval_loss": 2.5031485557556152,
+      "eval_runtime": 36.8147,
+      "eval_samples_per_second": 27.163,
+      "eval_steps_per_second": 1.141,
+      "step": 110
+    },
+    {
+      "epoch": 0.12164383561643835,
+      "grad_norm": 4.396062850952148,
+      "learning_rate": 1.77007299270073e-06,
+      "loss": 2.4373,
+      "step": 111
+    },
+    {
+      "epoch": 0.12273972602739726,
+      "grad_norm": 4.0441365242004395,
+      "learning_rate": 1.788321167883212e-06,
+      "loss": 2.4162,
+      "step": 112
+    },
+    {
+      "epoch": 0.12383561643835617,
+      "grad_norm": 2.8869969844818115,
+      "learning_rate": 1.8065693430656936e-06,
+      "loss": 2.4515,
+      "step": 113
+    },
+    {
+      "epoch": 0.12493150684931507,
+      "grad_norm": 3.4833669662475586,
+      "learning_rate": 1.8248175182481753e-06,
+      "loss": 2.3965,
+      "step": 114
+    },
+    {
+      "epoch": 0.12602739726027398,
+      "grad_norm": 2.9996068477630615,
+      "learning_rate": 1.8430656934306571e-06,
+      "loss": 2.3374,
+      "step": 115
+    },
+    {
+      "epoch": 0.12712328767123288,
+      "grad_norm": 2.701371192932129,
+      "learning_rate": 1.8613138686131388e-06,
+      "loss": 2.3608,
+      "step": 116
+    },
+    {
+      "epoch": 0.12821917808219177,
+      "grad_norm": 2.9491288661956787,
+      "learning_rate": 1.8795620437956206e-06,
+      "loss": 2.355,
+      "step": 117
+    },
+    {
+      "epoch": 0.1293150684931507,
+      "grad_norm": 2.9266560077667236,
+      "learning_rate": 1.8978102189781023e-06,
+      "loss": 2.3475,
+      "step": 118
+    },
+    {
+      "epoch": 0.1304109589041096,
+      "grad_norm": 3.0419797897338867,
+      "learning_rate": 1.916058394160584e-06,
+      "loss": 2.3475,
+      "step": 119
+    },
+    {
+      "epoch": 0.13150684931506848,
+      "grad_norm": 2.4531214237213135,
+      "learning_rate": 1.934306569343066e-06,
+      "loss": 2.3724,
+      "step": 120
+    },
+    {
+      "epoch": 0.13150684931506848,
+      "eval_loss": 2.455423593521118,
+      "eval_runtime": 37.0304,
+      "eval_samples_per_second": 27.005,
+      "eval_steps_per_second": 1.134,
+      "step": 120
+    },
+    {
+      "epoch": 0.1326027397260274,
+      "grad_norm": 3.572502851486206,
+      "learning_rate": 1.952554744525548e-06,
+      "loss": 2.2671,
+      "step": 121
+    },
+    {
+      "epoch": 0.1336986301369863,
+      "grad_norm": 3.5408504009246826,
+      "learning_rate": 1.9708029197080293e-06,
+      "loss": 2.3133,
+      "step": 122
+    },
+    {
+      "epoch": 0.13479452054794522,
+      "grad_norm": 2.488725423812866,
+      "learning_rate": 1.989051094890511e-06,
+      "loss": 2.3141,
+      "step": 123
+    },
+    {
+      "epoch": 0.1358904109589041,
+      "grad_norm": 3.241276264190674,
+      "learning_rate": 2.007299270072993e-06,
+      "loss": 2.318,
+      "step": 124
+    },
+    {
+      "epoch": 0.136986301369863,
+      "grad_norm": 2.853959798812866,
+      "learning_rate": 2.025547445255475e-06,
+      "loss": 2.3712,
+      "step": 125
+    },
+    {
+      "epoch": 0.13808219178082193,
+      "grad_norm": 2.9798169136047363,
+      "learning_rate": 2.0437956204379563e-06,
+      "loss": 2.3072,
+      "step": 126
+    },
+    {
+      "epoch": 0.13917808219178082,
+      "grad_norm": 2.8529601097106934,
+      "learning_rate": 2.062043795620438e-06,
+      "loss": 2.318,
+      "step": 127
+    },
+    {
+      "epoch": 0.14027397260273972,
+      "grad_norm": 3.475337028503418,
+      "learning_rate": 2.08029197080292e-06,
+      "loss": 2.31,
+      "step": 128
+    },
+    {
+      "epoch": 0.14136986301369864,
+      "grad_norm": 4.213955879211426,
+      "learning_rate": 2.0985401459854014e-06,
+      "loss": 2.3734,
+      "step": 129
+    },
+    {
+      "epoch": 0.14246575342465753,
+      "grad_norm": 3.0833330154418945,
+      "learning_rate": 2.1167883211678833e-06,
+      "loss": 2.2602,
+      "step": 130
+    },
+    {
+      "epoch": 0.14246575342465753,
+      "eval_loss": 2.4205682277679443,
+      "eval_runtime": 37.0798,
+      "eval_samples_per_second": 26.969,
+      "eval_steps_per_second": 1.133,
+      "step": 130
+    },
+    {
+      "epoch": 0.14356164383561643,
+      "grad_norm": 3.411235809326172,
+      "learning_rate": 2.135036496350365e-06,
+      "loss": 2.3255,
+      "step": 131
+    },
+    {
+      "epoch": 0.14465753424657535,
+      "grad_norm": 2.882053852081299,
+      "learning_rate": 2.1532846715328466e-06,
+      "loss": 2.3483,
+      "step": 132
+    },
+    {
+      "epoch": 0.14575342465753424,
+      "grad_norm": 3.2436420917510986,
+      "learning_rate": 2.171532846715329e-06,
+      "loss": 2.2492,
+      "step": 133
+    },
+    {
+      "epoch": 0.14684931506849316,
+      "grad_norm": 2.929466962814331,
+      "learning_rate": 2.1897810218978103e-06,
+      "loss": 2.2647,
+      "step": 134
+    },
+    {
+      "epoch": 0.14794520547945206,
+      "grad_norm": 2.9344966411590576,
+      "learning_rate": 2.208029197080292e-06,
+      "loss": 2.3061,
+      "step": 135
+    },
+    {
+      "epoch": 0.14904109589041095,
+      "grad_norm": 3.2567203044891357,
+      "learning_rate": 2.226277372262774e-06,
+      "loss": 2.2746,
+      "step": 136
+    },
+    {
+      "epoch": 0.15013698630136987,
+      "grad_norm": 3.1494054794311523,
+      "learning_rate": 2.244525547445256e-06,
+      "loss": 2.2866,
+      "step": 137
+    },
+    {
+      "epoch": 0.15123287671232877,
+      "grad_norm": 2.723390817642212,
+      "learning_rate": 2.2627737226277373e-06,
+      "loss": 2.3109,
+      "step": 138
+    },
+    {
+      "epoch": 0.15232876712328766,
+      "grad_norm": 2.8904967308044434,
+      "learning_rate": 2.281021897810219e-06,
+      "loss": 2.301,
+      "step": 139
+    },
+    {
+      "epoch": 0.15342465753424658,
+      "grad_norm": 2.8899483680725098,
+      "learning_rate": 2.299270072992701e-06,
+      "loss": 2.2637,
+      "step": 140
+    },
+    {
+      "epoch": 0.15342465753424658,
+      "eval_loss": 2.3952724933624268,
+      "eval_runtime": 37.1272,
+      "eval_samples_per_second": 26.934,
+      "eval_steps_per_second": 1.131,
+      "step": 140
+    },
+    {
+      "epoch": 0.15452054794520548,
+      "grad_norm": 3.5309066772460938,
+      "learning_rate": 2.3175182481751825e-06,
+      "loss": 2.2572,
+      "step": 141
+    },
+    {
+      "epoch": 0.15561643835616437,
+      "grad_norm": 2.8369667530059814,
+      "learning_rate": 2.3357664233576643e-06,
+      "loss": 2.2712,
+      "step": 142
+    },
+    {
+      "epoch": 0.1567123287671233,
+      "grad_norm": 3.629161834716797,
+      "learning_rate": 2.354014598540146e-06,
+      "loss": 2.2301,
+      "step": 143
+    },
+    {
+      "epoch": 0.1578082191780822,
+      "grad_norm": 3.5313000679016113,
+      "learning_rate": 2.3722627737226276e-06,
+      "loss": 2.2712,
+      "step": 144
+    },
+    {
+      "epoch": 0.1589041095890411,
+      "grad_norm": 3.1328771114349365,
+      "learning_rate": 2.39051094890511e-06,
+      "loss": 2.2706,
+      "step": 145
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.6992950439453125,
+      "learning_rate": 2.4087591240875918e-06,
+      "loss": 2.219,
+      "step": 146
+    },
+    {
+      "epoch": 0.1610958904109589,
+      "grad_norm": 5.230221748352051,
+      "learning_rate": 2.427007299270073e-06,
+      "loss": 2.2396,
+      "step": 147
+    },
+    {
+      "epoch": 0.16219178082191782,
+      "grad_norm": 3.5680761337280273,
+      "learning_rate": 2.445255474452555e-06,
+      "loss": 2.2434,
+      "step": 148
+    },
+    {
+      "epoch": 0.1632876712328767,
+      "grad_norm": 3.1642441749572754,
+      "learning_rate": 2.463503649635037e-06,
+      "loss": 2.2856,
+      "step": 149
+    },
+    {
+      "epoch": 0.1643835616438356,
+      "grad_norm": 2.694085121154785,
+      "learning_rate": 2.4817518248175183e-06,
+      "loss": 2.2006,
+      "step": 150
+    },
+    {
+      "epoch": 0.1643835616438356,
+      "eval_loss": 2.37199068069458,
+      "eval_runtime": 37.1342,
+      "eval_samples_per_second": 26.929,
+      "eval_steps_per_second": 1.131,
+      "step": 150
+    },
+    {
+      "epoch": 0.16547945205479453,
+      "grad_norm": 2.9868853092193604,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1694,
+      "step": 151
+    },
+    {
+      "epoch": 0.16657534246575342,
+      "grad_norm": 2.793532371520996,
+      "learning_rate": 2.518248175182482e-06,
+      "loss": 2.2965,
+      "step": 152
+    },
+    {
+      "epoch": 0.16767123287671232,
+      "grad_norm": 2.60152530670166,
+      "learning_rate": 2.5364963503649635e-06,
+      "loss": 2.1954,
+      "step": 153
+    },
+    {
+      "epoch": 0.16876712328767124,
+      "grad_norm": 2.6192219257354736,
+      "learning_rate": 2.5547445255474458e-06,
+      "loss": 2.1909,
+      "step": 154
+    },
+    {
+      "epoch": 0.16986301369863013,
+      "grad_norm": 2.467344045639038,
+      "learning_rate": 2.5729927007299272e-06,
+      "loss": 2.2438,
+      "step": 155
+    },
+    {
+      "epoch": 0.17095890410958905,
+      "grad_norm": 2.772012948989868,
+      "learning_rate": 2.591240875912409e-06,
+      "loss": 2.2213,
+      "step": 156
+    },
+    {
+      "epoch": 0.17205479452054795,
+      "grad_norm": 2.613675355911255,
+      "learning_rate": 2.6094890510948905e-06,
+      "loss": 2.2213,
+      "step": 157
+    },
+    {
+      "epoch": 0.17315068493150684,
+      "grad_norm": 2.849245071411133,
+      "learning_rate": 2.627737226277373e-06,
+      "loss": 2.255,
+      "step": 158
+    },
+    {
+      "epoch": 0.17424657534246576,
+      "grad_norm": 2.8388969898223877,
+      "learning_rate": 2.6459854014598542e-06,
+      "loss": 2.2572,
+      "step": 159
+    },
+    {
+      "epoch": 0.17534246575342466,
+      "grad_norm": 2.5219321250915527,
+      "learning_rate": 2.664233576642336e-06,
+      "loss": 2.2253,
+      "step": 160
+    },
+    {
+      "epoch": 0.17534246575342466,
+      "eval_loss": 2.3509061336517334,
+      "eval_runtime": 37.0953,
+      "eval_samples_per_second": 26.958,
+      "eval_steps_per_second": 1.132,
+      "step": 160
+    },
+    {
+      "epoch": 0.17643835616438355,
+      "grad_norm": 2.8460891246795654,
+      "learning_rate": 2.6824817518248175e-06,
+      "loss": 2.2009,
+      "step": 161
+    },
+    {
+      "epoch": 0.17753424657534247,
+      "grad_norm": 2.673377275466919,
+      "learning_rate": 2.7007299270072994e-06,
+      "loss": 2.2211,
+      "step": 162
+    },
+    {
+      "epoch": 0.17863013698630137,
+      "grad_norm": 2.8514182567596436,
+      "learning_rate": 2.718978102189781e-06,
+      "loss": 2.1955,
+      "step": 163
+    },
+    {
+      "epoch": 0.17972602739726026,
+      "grad_norm": 3.530385971069336,
+      "learning_rate": 2.737226277372263e-06,
+      "loss": 2.2536,
+      "step": 164
+    },
+    {
+      "epoch": 0.18082191780821918,
+      "grad_norm": 2.733394145965576,
+      "learning_rate": 2.755474452554745e-06,
+      "loss": 2.2367,
+      "step": 165
+    },
+    {
+      "epoch": 0.18191780821917808,
+      "grad_norm": 3.871150016784668,
+      "learning_rate": 2.7737226277372264e-06,
+      "loss": 2.1949,
+      "step": 166
+    },
+    {
+      "epoch": 0.18301369863013697,
+      "grad_norm": 3.7782530784606934,
+      "learning_rate": 2.7919708029197082e-06,
+      "loss": 2.2979,
+      "step": 167
+    },
+    {
+      "epoch": 0.1841095890410959,
+      "grad_norm": 3.094599485397339,
+      "learning_rate": 2.81021897810219e-06,
+      "loss": 2.2512,
+      "step": 168
+    },
+    {
+      "epoch": 0.1852054794520548,
+      "grad_norm": 3.2166848182678223,
+      "learning_rate": 2.828467153284672e-06,
+      "loss": 2.1784,
+      "step": 169
+    },
+    {
+      "epoch": 0.1863013698630137,
+      "grad_norm": 3.4127073287963867,
+      "learning_rate": 2.8467153284671534e-06,
+      "loss": 2.2241,
+      "step": 170
+    },
+    {
+      "epoch": 0.1863013698630137,
+      "eval_loss": 2.333245038986206,
+      "eval_runtime": 37.0402,
+      "eval_samples_per_second": 26.998,
+      "eval_steps_per_second": 1.134,
+      "step": 170
+    },
+    {
+      "epoch": 0.1873972602739726,
+      "grad_norm": 2.677579879760742,
+      "learning_rate": 2.8649635036496353e-06,
+      "loss": 2.135,
+      "step": 171
+    },
+    {
+      "epoch": 0.1884931506849315,
+      "grad_norm": 2.9602150917053223,
+      "learning_rate": 2.8832116788321167e-06,
+      "loss": 2.1916,
+      "step": 172
+    },
+    {
+      "epoch": 0.18958904109589042,
+      "grad_norm": 2.580122232437134,
+      "learning_rate": 2.901459854014599e-06,
+      "loss": 2.1477,
+      "step": 173
+    },
+    {
+      "epoch": 0.19068493150684931,
+      "grad_norm": 2.652217388153076,
+      "learning_rate": 2.9197080291970804e-06,
+      "loss": 2.2394,
+      "step": 174
+    },
+    {
+      "epoch": 0.1917808219178082,
+      "grad_norm": 2.6887567043304443,
+      "learning_rate": 2.9379562043795623e-06,
+      "loss": 2.2245,
+      "step": 175
+    },
+    {
+      "epoch": 0.19287671232876713,
+      "grad_norm": 2.768165111541748,
+      "learning_rate": 2.956204379562044e-06,
+      "loss": 2.2017,
+      "step": 176
+    },
+    {
+      "epoch": 0.19397260273972602,
+      "grad_norm": 2.9375691413879395,
+      "learning_rate": 2.9744525547445256e-06,
+      "loss": 2.2464,
+      "step": 177
+    },
+    {
+      "epoch": 0.19506849315068492,
+      "grad_norm": 2.8364248275756836,
+      "learning_rate": 2.992700729927008e-06,
+      "loss": 2.2155,
+      "step": 178
+    },
+    {
+      "epoch": 0.19616438356164384,
+      "grad_norm": 2.843487024307251,
+      "learning_rate": 3.0109489051094893e-06,
+      "loss": 2.1577,
+      "step": 179
+    },
+    {
+      "epoch": 0.19726027397260273,
+      "grad_norm": 3.005230665206909,
+      "learning_rate": 3.029197080291971e-06,
+      "loss": 2.1549,
+      "step": 180
+    },
+    {
+      "epoch": 0.19726027397260273,
+      "eval_loss": 2.3209409713745117,
+      "eval_runtime": 37.022,
+      "eval_samples_per_second": 27.011,
+      "eval_steps_per_second": 1.134,
+      "step": 180
+    },
+    {
+      "epoch": 0.19835616438356166,
+      "grad_norm": 2.490260601043701,
+      "learning_rate": 3.0474452554744526e-06,
+      "loss": 2.2313,
+      "step": 181
+    },
+    {
+      "epoch": 0.19945205479452055,
+      "grad_norm": 2.973118543624878,
+      "learning_rate": 3.065693430656935e-06,
+      "loss": 2.2316,
+      "step": 182
+    },
+    {
+      "epoch": 0.20054794520547944,
+      "grad_norm": 3.3308680057525635,
+      "learning_rate": 3.0839416058394163e-06,
+      "loss": 2.2087,
+      "step": 183
+    },
+    {
+      "epoch": 0.20164383561643837,
+      "grad_norm": 3.1440727710723877,
+      "learning_rate": 3.102189781021898e-06,
+      "loss": 2.2256,
+      "step": 184
+    },
+    {
+      "epoch": 0.20273972602739726,
+      "grad_norm": 2.5696325302124023,
+      "learning_rate": 3.1204379562043796e-06,
+      "loss": 2.2093,
+      "step": 185
+    },
+    {
+      "epoch": 0.20383561643835615,
+      "grad_norm": 3.1225037574768066,
+      "learning_rate": 3.1386861313868614e-06,
+      "loss": 2.1425,
+      "step": 186
+    },
+    {
+      "epoch": 0.20493150684931508,
+      "grad_norm": 3.650743246078491,
+      "learning_rate": 3.156934306569343e-06,
+      "loss": 2.1339,
+      "step": 187
+    },
+    {
+      "epoch": 0.20602739726027397,
+      "grad_norm": 2.924072027206421,
+      "learning_rate": 3.175182481751825e-06,
+      "loss": 2.2363,
+      "step": 188
+    },
+    {
+      "epoch": 0.20712328767123286,
+      "grad_norm": 2.9561989307403564,
+      "learning_rate": 3.193430656934307e-06,
+      "loss": 2.1934,
+      "step": 189
+    },
+    {
+      "epoch": 0.20821917808219179,
+      "grad_norm": 2.731393814086914,
+      "learning_rate": 3.2116788321167884e-06,
+      "loss": 2.1718,
+      "step": 190
+    },
+    {
+      "epoch": 0.20821917808219179,
+      "eval_loss": 2.3087782859802246,
+      "eval_runtime": 37.0161,
+      "eval_samples_per_second": 27.015,
+      "eval_steps_per_second": 1.135,
+      "step": 190
+    },
+    {
+      "epoch": 0.20931506849315068,
+      "grad_norm": 2.774442434310913,
+      "learning_rate": 3.2299270072992707e-06,
+      "loss": 2.1841,
+      "step": 191
+    },
+    {
+      "epoch": 0.2104109589041096,
+      "grad_norm": 3.0579135417938232,
+      "learning_rate": 3.248175182481752e-06,
+      "loss": 2.1677,
+      "step": 192
+    },
+    {
+      "epoch": 0.2115068493150685,
+      "grad_norm": 2.732984781265259,
+      "learning_rate": 3.266423357664234e-06,
+      "loss": 2.0975,
+      "step": 193
+    },
+    {
+      "epoch": 0.2126027397260274,
+      "grad_norm": 3.442659854888916,
+      "learning_rate": 3.2846715328467155e-06,
+      "loss": 2.1923,
+      "step": 194
+    },
+    {
+      "epoch": 0.2136986301369863,
+      "grad_norm": 3.064223527908325,
+      "learning_rate": 3.3029197080291973e-06,
+      "loss": 2.1707,
+      "step": 195
+    },
+    {
+      "epoch": 0.2147945205479452,
+      "grad_norm": 2.5138468742370605,
+      "learning_rate": 3.3211678832116788e-06,
+      "loss": 2.1933,
+      "step": 196
+    },
+    {
+      "epoch": 0.2158904109589041,
+      "grad_norm": 3.1076464653015137,
+      "learning_rate": 3.339416058394161e-06,
+      "loss": 2.1725,
+      "step": 197
+    },
+    {
+      "epoch": 0.21698630136986302,
+      "grad_norm": 2.7291388511657715,
+      "learning_rate": 3.3576642335766425e-06,
+      "loss": 2.1722,
+      "step": 198
+    },
+    {
+      "epoch": 0.21808219178082192,
+      "grad_norm": 2.805506467819214,
+      "learning_rate": 3.3759124087591243e-06,
+      "loss": 2.1688,
+      "step": 199
+    },
+    {
+      "epoch": 0.2191780821917808,
+      "grad_norm": 2.6190085411071777,
+      "learning_rate": 3.3941605839416058e-06,
+      "loss": 2.1708,
+      "step": 200
+    },
+    {
+      "epoch": 0.2191780821917808,
+      "eval_loss": 2.295546054840088,
+      "eval_runtime": 37.0133,
+      "eval_samples_per_second": 27.017,
+      "eval_steps_per_second": 1.135,
+      "step": 200
+    },
+    {
+      "epoch": 0.22027397260273973,
+      "grad_norm": 3.1270673274993896,
+      "learning_rate": 3.412408759124088e-06,
+      "loss": 2.1759,
+      "step": 201
+    },
+    {
+      "epoch": 0.22136986301369863,
+      "grad_norm": 3.494178533554077,
+      "learning_rate": 3.43065693430657e-06,
+      "loss": 2.1795,
+      "step": 202
+    },
+    {
+      "epoch": 0.22246575342465755,
+      "grad_norm": 2.7398736476898193,
+      "learning_rate": 3.4489051094890513e-06,
+      "loss": 2.1201,
+      "step": 203
+    },
+    {
+      "epoch": 0.22356164383561644,
+      "grad_norm": 2.981506824493408,
+      "learning_rate": 3.467153284671533e-06,
+      "loss": 2.1693,
+      "step": 204
+    },
+    {
+      "epoch": 0.22465753424657534,
+      "grad_norm": 2.4129178524017334,
+      "learning_rate": 3.4854014598540146e-06,
+      "loss": 2.1125,
+      "step": 205
+    },
+    {
+      "epoch": 0.22575342465753426,
+      "grad_norm": 2.701197385787964,
+      "learning_rate": 3.503649635036497e-06,
+      "loss": 2.1565,
+      "step": 206
+    },
+    {
+      "epoch": 0.22684931506849315,
+      "grad_norm": 2.533306837081909,
+      "learning_rate": 3.5218978102189783e-06,
+      "loss": 2.1487,
+      "step": 207
+    },
+    {
+      "epoch": 0.22794520547945205,
+      "grad_norm": 2.8030788898468018,
+      "learning_rate": 3.54014598540146e-06,
+      "loss": 2.1801,
+      "step": 208
+    },
+    {
+      "epoch": 0.22904109589041097,
+      "grad_norm": 2.895414352416992,
+      "learning_rate": 3.5583941605839416e-06,
+      "loss": 2.189,
+      "step": 209
+    },
+    {
+      "epoch": 0.23013698630136986,
+      "grad_norm": 2.508390426635742,
+      "learning_rate": 3.576642335766424e-06,
+      "loss": 2.2376,
+      "step": 210
+    },
+    {
+      "epoch": 0.23013698630136986,
+      "eval_loss": 2.2900516986846924,
+      "eval_runtime": 36.7645,
+      "eval_samples_per_second": 27.2,
+      "eval_steps_per_second": 1.142,
+      "step": 210
+    },
+    {
+      "epoch": 0.23123287671232876,
+      "grad_norm": 2.6393816471099854,
+      "learning_rate": 3.5948905109489054e-06,
+      "loss": 2.2267,
+      "step": 211
+    },
+    {
+      "epoch": 0.23232876712328768,
+      "grad_norm": 3.0687549114227295,
+      "learning_rate": 3.6131386861313872e-06,
+      "loss": 2.0744,
+      "step": 212
+    },
+    {
+      "epoch": 0.23342465753424657,
+      "grad_norm": 2.6342105865478516,
+      "learning_rate": 3.6313868613138687e-06,
+      "loss": 2.0737,
+      "step": 213
+    },
+    {
+      "epoch": 0.2345205479452055,
+      "grad_norm": 3.5662875175476074,
+      "learning_rate": 3.6496350364963505e-06,
+      "loss": 2.1147,
+      "step": 214
+    },
+    {
+      "epoch": 0.2356164383561644,
+      "grad_norm": 2.5357561111450195,
+      "learning_rate": 3.667883211678833e-06,
+      "loss": 2.1268,
+      "step": 215
+    },
+    {
+      "epoch": 0.23671232876712328,
+      "grad_norm": 3.439711093902588,
+      "learning_rate": 3.6861313868613142e-06,
+      "loss": 2.1118,
+      "step": 216
+    },
+    {
+      "epoch": 0.2378082191780822,
+      "grad_norm": 2.9410204887390137,
+      "learning_rate": 3.704379562043796e-06,
+      "loss": 2.1087,
+      "step": 217
+    },
+    {
+      "epoch": 0.2389041095890411,
+      "grad_norm": 3.3182294368743896,
+      "learning_rate": 3.7226277372262775e-06,
+      "loss": 2.196,
+      "step": 218
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 3.3834011554718018,
+      "learning_rate": 3.7408759124087594e-06,
+      "loss": 2.1633,
+      "step": 219
+    },
+    {
+      "epoch": 0.2410958904109589,
+      "grad_norm": 3.114952325820923,
+      "learning_rate": 3.7591240875912412e-06,
+      "loss": 2.1107,
+      "step": 220
+    },
+    {
+      "epoch": 0.2410958904109589,
+      "eval_loss": 2.277580976486206,
+      "eval_runtime": 36.9846,
+      "eval_samples_per_second": 27.038,
+      "eval_steps_per_second": 1.136,
+      "step": 220
+    },
+    {
+      "epoch": 0.2421917808219178,
+      "grad_norm": 3.1151392459869385,
+      "learning_rate": 3.777372262773723e-06,
+      "loss": 2.1471,
+      "step": 221
+    },
+    {
+      "epoch": 0.2432876712328767,
+      "grad_norm": 3.798450469970703,
+      "learning_rate": 3.7956204379562045e-06,
+      "loss": 2.1324,
+      "step": 222
+    },
+    {
+      "epoch": 0.24438356164383562,
+      "grad_norm": 2.6187098026275635,
+      "learning_rate": 3.8138686131386864e-06,
+      "loss": 2.1061,
+      "step": 223
+    },
+    {
+      "epoch": 0.24547945205479452,
+      "grad_norm": 4.014018535614014,
+      "learning_rate": 3.832116788321168e-06,
+      "loss": 2.166,
+      "step": 224
+    },
+    {
+      "epoch": 0.2465753424657534,
+      "grad_norm": 3.0043938159942627,
+      "learning_rate": 3.85036496350365e-06,
+      "loss": 2.1197,
+      "step": 225
+    },
+    {
+      "epoch": 0.24767123287671233,
+      "grad_norm": 3.507066249847412,
+      "learning_rate": 3.868613138686132e-06,
+      "loss": 2.0848,
+      "step": 226
+    },
+    {
+      "epoch": 0.24876712328767123,
+      "grad_norm": 3.5713281631469727,
+      "learning_rate": 3.886861313868613e-06,
+      "loss": 2.1239,
+      "step": 227
+    },
+    {
+      "epoch": 0.24986301369863015,
+      "grad_norm": 3.961082696914673,
+      "learning_rate": 3.905109489051096e-06,
+      "loss": 2.1406,
+      "step": 228
+    },
+    {
+      "epoch": 0.25095890410958904,
+      "grad_norm": 3.1514859199523926,
+      "learning_rate": 3.923357664233577e-06,
+      "loss": 2.1689,
+      "step": 229
+    },
+    {
+      "epoch": 0.25205479452054796,
+      "grad_norm": 3.8486833572387695,
+      "learning_rate": 3.9416058394160585e-06,
+      "loss": 2.1321,
+      "step": 230
+    },
+    {
+      "epoch": 0.25205479452054796,
+      "eval_loss": 2.2620601654052734,
+      "eval_runtime": 37.058,
+      "eval_samples_per_second": 26.985,
+      "eval_steps_per_second": 1.133,
+      "step": 230
+    },
+    {
+      "epoch": 0.25315068493150683,
+      "grad_norm": 2.5626907348632812,
+      "learning_rate": 3.95985401459854e-06,
+      "loss": 2.1777,
+      "step": 231
+    },
+    {
+      "epoch": 0.25424657534246575,
+      "grad_norm": 4.105278015136719,
+      "learning_rate": 3.978102189781022e-06,
+      "loss": 2.2099,
+      "step": 232
+    },
+    {
+      "epoch": 0.2553424657534247,
+      "grad_norm": 2.9818739891052246,
+      "learning_rate": 3.996350364963504e-06,
+      "loss": 2.0705,
+      "step": 233
+    },
+    {
+      "epoch": 0.25643835616438354,
+      "grad_norm": 2.87030291557312,
+      "learning_rate": 4.014598540145986e-06,
+      "loss": 2.1453,
+      "step": 234
+    },
+    {
+      "epoch": 0.25753424657534246,
+      "grad_norm": 2.4847915172576904,
+      "learning_rate": 4.032846715328467e-06,
+      "loss": 2.0461,
+      "step": 235
+    },
+    {
+      "epoch": 0.2586301369863014,
+      "grad_norm": 3.451420783996582,
+      "learning_rate": 4.05109489051095e-06,
+      "loss": 2.1133,
+      "step": 236
+    },
+    {
+      "epoch": 0.25972602739726025,
+      "grad_norm": 2.954723358154297,
+      "learning_rate": 4.069343065693431e-06,
+      "loss": 2.1316,
+      "step": 237
+    },
+    {
+      "epoch": 0.2608219178082192,
+      "grad_norm": 2.5295896530151367,
+      "learning_rate": 4.0875912408759126e-06,
+      "loss": 2.0901,
+      "step": 238
+    },
+    {
+      "epoch": 0.2619178082191781,
+      "grad_norm": 2.6359453201293945,
+      "learning_rate": 4.105839416058395e-06,
+      "loss": 2.117,
+      "step": 239
+    },
+    {
+      "epoch": 0.26301369863013696,
+      "grad_norm": 2.5236105918884277,
+      "learning_rate": 4.124087591240876e-06,
+      "loss": 2.1958,
+      "step": 240
+    },
+    {
+      "epoch": 0.26301369863013696,
+      "eval_loss": 2.250295400619507,
+      "eval_runtime": 37.0811,
+      "eval_samples_per_second": 26.968,
+      "eval_steps_per_second": 1.133,
+      "step": 240
+    },
+    {
+      "epoch": 0.2641095890410959,
+      "grad_norm": 2.581918954849243,
+      "learning_rate": 4.1423357664233586e-06,
+      "loss": 2.1054,
+      "step": 241
+    },
+    {
+      "epoch": 0.2652054794520548,
+      "grad_norm": 2.501332998275757,
+      "learning_rate": 4.16058394160584e-06,
+      "loss": 2.0878,
+      "step": 242
+    },
+    {
+      "epoch": 0.2663013698630137,
+      "grad_norm": 2.6477770805358887,
+      "learning_rate": 4.1788321167883214e-06,
+      "loss": 2.1656,
+      "step": 243
+    },
+    {
+      "epoch": 0.2673972602739726,
+      "grad_norm": 2.7252438068389893,
+      "learning_rate": 4.197080291970803e-06,
+      "loss": 2.0497,
+      "step": 244
+    },
+    {
+      "epoch": 0.2684931506849315,
+      "grad_norm": 2.6056301593780518,
+      "learning_rate": 4.215328467153285e-06,
+      "loss": 2.1411,
+      "step": 245
+    },
+    {
+      "epoch": 0.26958904109589044,
+      "grad_norm": 2.6581931114196777,
+      "learning_rate": 4.233576642335767e-06,
+      "loss": 2.0829,
+      "step": 246
+    },
+    {
+      "epoch": 0.2706849315068493,
+      "grad_norm": 2.6316752433776855,
+      "learning_rate": 4.251824817518249e-06,
+      "loss": 2.0882,
+      "step": 247
+    },
+    {
+      "epoch": 0.2717808219178082,
+      "grad_norm": 2.803823947906494,
+      "learning_rate": 4.27007299270073e-06,
+      "loss": 2.1203,
+      "step": 248
+    },
+    {
+      "epoch": 0.27287671232876715,
+      "grad_norm": 2.9551174640655518,
+      "learning_rate": 4.288321167883212e-06,
+      "loss": 2.0897,
+      "step": 249
+    },
+    {
+      "epoch": 0.273972602739726,
+      "grad_norm": 2.4847161769866943,
+      "learning_rate": 4.306569343065693e-06,
+      "loss": 2.0845,
+      "step": 250
+    },
+    {
+      "epoch": 0.273972602739726,
+      "eval_loss": 2.2491259574890137,
+      "eval_runtime": 37.1024,
+      "eval_samples_per_second": 26.952,
+      "eval_steps_per_second": 1.132,
+      "step": 250
+    },
+    {
+      "epoch": 0.27506849315068493,
+      "grad_norm": 2.6398351192474365,
+      "learning_rate": 4.3248175182481755e-06,
+      "loss": 2.0479,
+      "step": 251
+    },
+    {
+      "epoch": 0.27616438356164386,
+      "grad_norm": 3.0948562622070312,
+      "learning_rate": 4.343065693430658e-06,
+      "loss": 2.1184,
+      "step": 252
+    },
+    {
+      "epoch": 0.2772602739726027,
+      "grad_norm": 2.5378012657165527,
+      "learning_rate": 4.361313868613139e-06,
+      "loss": 2.1099,
+      "step": 253
+    },
+    {
+      "epoch": 0.27835616438356164,
+      "grad_norm": 3.0282225608825684,
+      "learning_rate": 4.379562043795621e-06,
+      "loss": 2.0982,
+      "step": 254
+    },
+    {
+      "epoch": 0.27945205479452057,
+      "grad_norm": 2.6205334663391113,
+      "learning_rate": 4.397810218978102e-06,
+      "loss": 2.1285,
+      "step": 255
+    },
+    {
+      "epoch": 0.28054794520547943,
+      "grad_norm": 3.2056095600128174,
+      "learning_rate": 4.416058394160584e-06,
+      "loss": 2.1221,
+      "step": 256
+    },
+    {
+      "epoch": 0.28164383561643835,
+      "grad_norm": 3.1370127201080322,
+      "learning_rate": 4.434306569343066e-06,
+      "loss": 2.1016,
+      "step": 257
+    },
+    {
+      "epoch": 0.2827397260273973,
+      "grad_norm": 2.494168519973755,
+      "learning_rate": 4.452554744525548e-06,
+      "loss": 2.0754,
+      "step": 258
+    },
+    {
+      "epoch": 0.28383561643835614,
+      "grad_norm": 2.7979183197021484,
+      "learning_rate": 4.4708029197080295e-06,
+      "loss": 2.1218,
+      "step": 259
+    },
+    {
+      "epoch": 0.28493150684931506,
+      "grad_norm": 2.755682945251465,
+      "learning_rate": 4.489051094890512e-06,
+      "loss": 2.1136,
+      "step": 260
+    },
+    {
+      "epoch": 0.28493150684931506,
+      "eval_loss": 2.2433817386627197,
+      "eval_runtime": 37.1121,
+      "eval_samples_per_second": 26.945,
+      "eval_steps_per_second": 1.132,
+      "step": 260
+    },
+    {
+      "epoch": 0.286027397260274,
+      "grad_norm": 2.3641891479492188,
+      "learning_rate": 4.507299270072993e-06,
+      "loss": 2.1488,
+      "step": 261
+    },
+    {
+      "epoch": 0.28712328767123285,
+      "grad_norm": 2.741292953491211,
+      "learning_rate": 4.525547445255475e-06,
+      "loss": 2.1048,
+      "step": 262
+    },
+    {
+      "epoch": 0.2882191780821918,
+      "grad_norm": 2.811483383178711,
+      "learning_rate": 4.543795620437956e-06,
+      "loss": 2.0996,
+      "step": 263
+    },
+    {
+      "epoch": 0.2893150684931507,
+      "grad_norm": 2.6705479621887207,
+      "learning_rate": 4.562043795620438e-06,
+      "loss": 2.0652,
+      "step": 264
+    },
+    {
+      "epoch": 0.29041095890410956,
+      "grad_norm": 2.9383747577667236,
+      "learning_rate": 4.580291970802921e-06,
+      "loss": 2.1574,
+      "step": 265
+    },
+    {
+      "epoch": 0.2915068493150685,
+      "grad_norm": 2.591373920440674,
+      "learning_rate": 4.598540145985402e-06,
+      "loss": 2.1655,
+      "step": 266
+    },
+    {
+      "epoch": 0.2926027397260274,
+      "grad_norm": 3.16955828666687,
+      "learning_rate": 4.6167883211678835e-06,
+      "loss": 2.1285,
+      "step": 267
+    },
+    {
+      "epoch": 0.2936986301369863,
+      "grad_norm": 2.66941499710083,
+      "learning_rate": 4.635036496350365e-06,
+      "loss": 2.0777,
+      "step": 268
+    },
+    {
+      "epoch": 0.2947945205479452,
+      "grad_norm": 2.768139600753784,
+      "learning_rate": 4.653284671532847e-06,
+      "loss": 2.0748,
+      "step": 269
+    },
+    {
+      "epoch": 0.2958904109589041,
+      "grad_norm": 2.5951924324035645,
+      "learning_rate": 4.671532846715329e-06,
+      "loss": 2.1289,
+      "step": 270
+    },
+    {
+      "epoch": 0.2958904109589041,
+      "eval_loss": 2.2393550872802734,
+      "eval_runtime": 37.0993,
+      "eval_samples_per_second": 26.955,
+      "eval_steps_per_second": 1.132,
+      "step": 270
+    },
+    {
+      "epoch": 0.29698630136986304,
+      "grad_norm": 2.5422160625457764,
+      "learning_rate": 4.689781021897811e-06,
+      "loss": 2.1342,
+      "step": 271
+    },
+    {
+      "epoch": 0.2980821917808219,
+      "grad_norm": 2.540555477142334,
+      "learning_rate": 4.708029197080292e-06,
+      "loss": 2.097,
+      "step": 272
+    },
+    {
+      "epoch": 0.2991780821917808,
+      "grad_norm": 2.995366334915161,
+      "learning_rate": 4.726277372262774e-06,
+      "loss": 2.1452,
+      "step": 273
+    },
+    {
+      "epoch": 0.30027397260273975,
+      "grad_norm": 2.2561020851135254,
+      "learning_rate": 4.744525547445255e-06,
+      "loss": 2.1069,
+      "step": 274
+    },
+    {
+      "epoch": 0.3013698630136986,
+      "grad_norm": 2.53476619720459,
+      "learning_rate": 4.7627737226277375e-06,
+      "loss": 2.0381,
+      "step": 275
+    },
+    {
+      "epoch": 0.30246575342465754,
+      "grad_norm": 2.3415749073028564,
+      "learning_rate": 4.78102189781022e-06,
+      "loss": 2.1027,
+      "step": 276
+    },
+    {
+      "epoch": 0.30356164383561646,
+      "grad_norm": 2.9756431579589844,
+      "learning_rate": 4.799270072992701e-06,
+      "loss": 2.1136,
+      "step": 277
+    },
+    {
+      "epoch": 0.3046575342465753,
+      "grad_norm": 2.606208324432373,
+      "learning_rate": 4.8175182481751835e-06,
+      "loss": 2.0797,
+      "step": 278
+    },
+    {
+      "epoch": 0.30575342465753425,
+      "grad_norm": 2.5089313983917236,
+      "learning_rate": 4.835766423357665e-06,
+      "loss": 2.1026,
+      "step": 279
+    },
+    {
+      "epoch": 0.30684931506849317,
+      "grad_norm": 2.5003838539123535,
+      "learning_rate": 4.854014598540146e-06,
+      "loss": 2.0923,
+      "step": 280
+    },
+    {
+      "epoch": 0.30684931506849317,
+      "eval_loss": 2.2351036071777344,
+      "eval_runtime": 37.111,
+      "eval_samples_per_second": 26.946,
+      "eval_steps_per_second": 1.132,
+      "step": 280
+    },
+    {
+      "epoch": 0.30794520547945203,
+      "grad_norm": 2.403602361679077,
+      "learning_rate": 4.872262773722628e-06,
+      "loss": 2.0388,
+      "step": 281
+    },
+    {
+      "epoch": 0.30904109589041096,
+      "grad_norm": 2.7775959968566895,
+      "learning_rate": 4.89051094890511e-06,
+      "loss": 2.0561,
+      "step": 282
+    },
+    {
+      "epoch": 0.3101369863013699,
+      "grad_norm": 2.7128653526306152,
+      "learning_rate": 4.9087591240875915e-06,
+      "loss": 2.042,
+      "step": 283
+    },
+    {
+      "epoch": 0.31123287671232874,
+      "grad_norm": 2.7029991149902344,
+      "learning_rate": 4.927007299270074e-06,
+      "loss": 2.0814,
+      "step": 284
+    },
+    {
+      "epoch": 0.31232876712328766,
+      "grad_norm": 2.3942933082580566,
+      "learning_rate": 4.945255474452555e-06,
+      "loss": 2.0906,
+      "step": 285
+    },
+    {
+      "epoch": 0.3134246575342466,
+      "grad_norm": 2.5086541175842285,
+      "learning_rate": 4.963503649635037e-06,
+      "loss": 2.0625,
+      "step": 286
+    },
+    {
+      "epoch": 0.31452054794520545,
+      "grad_norm": 2.6668760776519775,
+      "learning_rate": 4.981751824817518e-06,
+      "loss": 2.1538,
+      "step": 287
+    },
+    {
+      "epoch": 0.3156164383561644,
+      "grad_norm": 2.5066750049591064,
+      "learning_rate": 5e-06,
+      "loss": 2.1423,
+      "step": 288
+    },
+    {
+      "epoch": 0.3167123287671233,
+      "grad_norm": 2.84967041015625,
+      "learning_rate": 4.997969130787978e-06,
+      "loss": 2.0943,
+      "step": 289
+    },
+    {
+      "epoch": 0.3178082191780822,
+      "grad_norm": 2.151946544647217,
+      "learning_rate": 4.995938261575955e-06,
+      "loss": 2.1098,
+      "step": 290
+    },
+    {
+      "epoch": 0.3178082191780822,
+      "eval_loss": 2.223247528076172,
+      "eval_runtime": 37.1755,
+      "eval_samples_per_second": 26.899,
+      "eval_steps_per_second": 1.13,
+      "step": 290
+    },
+    {
+      "epoch": 0.3189041095890411,
+      "grad_norm": 2.887166976928711,
+      "learning_rate": 4.993907392363932e-06,
+      "loss": 2.0281,
+      "step": 291
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.5930211544036865,
+      "learning_rate": 4.9918765231519095e-06,
+      "loss": 2.1063,
+      "step": 292
+    },
+    {
+      "epoch": 0.32109589041095893,
+      "grad_norm": 2.5568413734436035,
+      "learning_rate": 4.989845653939887e-06,
+      "loss": 2.0313,
+      "step": 293
+    },
+    {
+      "epoch": 0.3221917808219178,
+      "grad_norm": 2.536095380783081,
+      "learning_rate": 4.987814784727864e-06,
+      "loss": 2.0749,
+      "step": 294
+    },
+    {
+      "epoch": 0.3232876712328767,
+      "grad_norm": 2.3385651111602783,
+      "learning_rate": 4.985783915515841e-06,
+      "loss": 2.0666,
+      "step": 295
+    },
+    {
+      "epoch": 0.32438356164383564,
+      "grad_norm": 2.504826068878174,
+      "learning_rate": 4.983753046303819e-06,
+      "loss": 2.0677,
+      "step": 296
+    },
+    {
+      "epoch": 0.3254794520547945,
+      "grad_norm": 2.5289409160614014,
+      "learning_rate": 4.981722177091796e-06,
+      "loss": 2.0828,
+      "step": 297
+    },
+    {
+      "epoch": 0.3265753424657534,
+      "grad_norm": 2.803795099258423,
+      "learning_rate": 4.979691307879773e-06,
+      "loss": 2.0733,
+      "step": 298
+    },
+    {
+      "epoch": 0.32767123287671235,
+      "grad_norm": 2.651273012161255,
+      "learning_rate": 4.9776604386677504e-06,
+      "loss": 2.0987,
+      "step": 299
+    },
+    {
+      "epoch": 0.3287671232876712,
+      "grad_norm": 2.4715864658355713,
+      "learning_rate": 4.975629569455728e-06,
+      "loss": 2.0733,
+      "step": 300
+    },
+    {
+      "epoch": 0.3287671232876712,
+      "eval_loss": 2.214858293533325,
+      "eval_runtime": 37.0559,
+      "eval_samples_per_second": 26.986,
+      "eval_steps_per_second": 1.133,
+      "step": 300
+    },
+    {
+      "epoch": 0.32986301369863014,
+      "grad_norm": 2.2573442459106445,
+      "learning_rate": 4.973598700243705e-06,
+      "loss": 2.1291,
+      "step": 301
+    },
+    {
+      "epoch": 0.33095890410958906,
+      "grad_norm": 2.645966053009033,
+      "learning_rate": 4.971567831031681e-06,
+      "loss": 2.046,
+      "step": 302
+    },
+    {
+      "epoch": 0.3320547945205479,
+      "grad_norm": 2.361392021179199,
+      "learning_rate": 4.9695369618196595e-06,
+      "loss": 2.0447,
+      "step": 303
+    },
+    {
+      "epoch": 0.33315068493150685,
+      "grad_norm": 2.272714376449585,
+      "learning_rate": 4.967506092607636e-06,
+      "loss": 1.9272,
+      "step": 304
+    },
+    {
+      "epoch": 0.33424657534246577,
+      "grad_norm": 2.5846943855285645,
+      "learning_rate": 4.965475223395614e-06,
+      "loss": 2.0199,
+      "step": 305
+    },
+    {
+      "epoch": 0.33534246575342463,
+      "grad_norm": 2.38985013961792,
+      "learning_rate": 4.9634443541835905e-06,
+      "loss": 2.0394,
+      "step": 306
+    },
+    {
+      "epoch": 0.33643835616438356,
+      "grad_norm": 2.3526251316070557,
+      "learning_rate": 4.961413484971569e-06,
+      "loss": 2.0403,
+      "step": 307
+    },
+    {
+      "epoch": 0.3375342465753425,
+      "grad_norm": 2.399531602859497,
+      "learning_rate": 4.959382615759545e-06,
+      "loss": 2.0607,
+      "step": 308
+    },
+    {
+      "epoch": 0.33863013698630134,
+      "grad_norm": 2.50803804397583,
+      "learning_rate": 4.957351746547523e-06,
+      "loss": 2.0688,
+      "step": 309
+    },
+    {
+      "epoch": 0.33972602739726027,
+      "grad_norm": 2.278088092803955,
+      "learning_rate": 4.9553208773355e-06,
+      "loss": 2.0453,
+      "step": 310
+    },
+    {
+      "epoch": 0.33972602739726027,
+      "eval_loss": 2.2185235023498535,
+      "eval_runtime": 36.9712,
+      "eval_samples_per_second": 27.048,
+      "eval_steps_per_second": 1.136,
+      "step": 310
+    },
+    {
+      "epoch": 0.3408219178082192,
+      "grad_norm": 2.555300712585449,
+      "learning_rate": 4.953290008123478e-06,
+      "loss": 2.1726,
+      "step": 311
+    },
+    {
+      "epoch": 0.3419178082191781,
+      "grad_norm": 2.614870548248291,
+      "learning_rate": 4.951259138911454e-06,
+      "loss": 2.0659,
+      "step": 312
+    },
+    {
+      "epoch": 0.343013698630137,
+      "grad_norm": 2.1932661533355713,
+      "learning_rate": 4.9492282696994315e-06,
+      "loss": 1.9981,
+      "step": 313
+    },
+    {
+      "epoch": 0.3441095890410959,
+      "grad_norm": 2.707451581954956,
+      "learning_rate": 4.947197400487409e-06,
+      "loss": 2.1321,
+      "step": 314
+    },
+    {
+      "epoch": 0.3452054794520548,
+      "grad_norm": 2.4317946434020996,
+      "learning_rate": 4.945166531275386e-06,
+      "loss": 2.0548,
+      "step": 315
+    },
+    {
+      "epoch": 0.3463013698630137,
+      "grad_norm": 2.639883279800415,
+      "learning_rate": 4.943135662063363e-06,
+      "loss": 2.0512,
+      "step": 316
+    },
+    {
+      "epoch": 0.3473972602739726,
+      "grad_norm": 3.1968822479248047,
+      "learning_rate": 4.9411047928513406e-06,
+      "loss": 2.0611,
+      "step": 317
+    },
+    {
+      "epoch": 0.34849315068493153,
+      "grad_norm": 2.4361510276794434,
+      "learning_rate": 4.939073923639318e-06,
+      "loss": 2.0974,
+      "step": 318
+    },
+    {
+      "epoch": 0.3495890410958904,
+      "grad_norm": 2.8278262615203857,
+      "learning_rate": 4.937043054427295e-06,
+      "loss": 2.0992,
+      "step": 319
+    },
+    {
+      "epoch": 0.3506849315068493,
+      "grad_norm": 3.01247239112854,
+      "learning_rate": 4.935012185215272e-06,
+      "loss": 2.0372,
+      "step": 320
+    },
+    {
+      "epoch": 0.3506849315068493,
+      "eval_loss": 2.206442356109619,
+      "eval_runtime": 36.8797,
+      "eval_samples_per_second": 27.115,
+      "eval_steps_per_second": 1.139,
+      "step": 320
+    },
+    {
+      "epoch": 0.35178082191780824,
+      "grad_norm": 2.3372156620025635,
+      "learning_rate": 4.93298131600325e-06,
+      "loss": 2.0211,
+      "step": 321
+    },
+    {
+      "epoch": 0.3528767123287671,
+      "grad_norm": 2.91377329826355,
+      "learning_rate": 4.930950446791227e-06,
+      "loss": 2.0879,
+      "step": 322
+    },
+    {
+      "epoch": 0.353972602739726,
+      "grad_norm": 2.3207666873931885,
+      "learning_rate": 4.928919577579204e-06,
+      "loss": 2.056,
+      "step": 323
+    },
+    {
+      "epoch": 0.35506849315068495,
+      "grad_norm": 2.3260514736175537,
+      "learning_rate": 4.9268887083671815e-06,
+      "loss": 2.0684,
+      "step": 324
+    },
+    {
+      "epoch": 0.3561643835616438,
+      "grad_norm": 2.3070554733276367,
+      "learning_rate": 4.924857839155159e-06,
+      "loss": 2.0494,
+      "step": 325
+    },
+    {
+      "epoch": 0.35726027397260274,
+      "grad_norm": 2.189871072769165,
+      "learning_rate": 4.922826969943136e-06,
+      "loss": 2.0531,
+      "step": 326
+    },
+    {
+      "epoch": 0.35835616438356166,
+      "grad_norm": 2.553520917892456,
+      "learning_rate": 4.920796100731113e-06,
+      "loss": 2.0749,
+      "step": 327
+    },
+    {
+      "epoch": 0.3594520547945205,
+      "grad_norm": 2.62296199798584,
+      "learning_rate": 4.918765231519091e-06,
+      "loss": 1.9983,
+      "step": 328
+    },
+    {
+      "epoch": 0.36054794520547945,
+      "grad_norm": 2.4612369537353516,
+      "learning_rate": 4.916734362307068e-06,
+      "loss": 2.0641,
+      "step": 329
+    },
+    {
+      "epoch": 0.36164383561643837,
+      "grad_norm": 2.656157970428467,
+      "learning_rate": 4.914703493095045e-06,
+      "loss": 2.0148,
+      "step": 330
+    },
+    {
+      "epoch": 0.36164383561643837,
+      "eval_loss": 2.2054214477539062,
+      "eval_runtime": 36.886,
+      "eval_samples_per_second": 27.111,
+      "eval_steps_per_second": 1.139,
+      "step": 330
+    },
+    {
+      "epoch": 0.36273972602739724,
+      "grad_norm": 2.2823736667633057,
+      "learning_rate": 4.912672623883022e-06,
+      "loss": 2.0221,
+      "step": 331
+    },
+    {
+      "epoch": 0.36383561643835616,
+      "grad_norm": 2.8403289318084717,
+      "learning_rate": 4.910641754671e-06,
+      "loss": 2.0149,
+      "step": 332
+    },
+    {
+      "epoch": 0.3649315068493151,
+      "grad_norm": 2.453887462615967,
+      "learning_rate": 4.908610885458977e-06,
+      "loss": 2.0411,
+      "step": 333
+    },
+    {
+      "epoch": 0.36602739726027395,
+      "grad_norm": 2.6094300746917725,
+      "learning_rate": 4.906580016246954e-06,
+      "loss": 2.0293,
+      "step": 334
+    },
+    {
+      "epoch": 0.36712328767123287,
+      "grad_norm": 2.4412384033203125,
+      "learning_rate": 4.9045491470349315e-06,
+      "loss": 2.0778,
+      "step": 335
+    },
+    {
+      "epoch": 0.3682191780821918,
+      "grad_norm": 2.4435057640075684,
+      "learning_rate": 4.902518277822909e-06,
+      "loss": 2.0597,
+      "step": 336
+    },
+    {
+      "epoch": 0.3693150684931507,
+      "grad_norm": 2.2390244007110596,
+      "learning_rate": 4.900487408610886e-06,
+      "loss": 2.0654,
+      "step": 337
+    },
+    {
+      "epoch": 0.3704109589041096,
+      "grad_norm": 2.4243953227996826,
+      "learning_rate": 4.898456539398863e-06,
+      "loss": 2.0696,
+      "step": 338
+    },
+    {
+      "epoch": 0.3715068493150685,
+      "grad_norm": 2.375720262527466,
+      "learning_rate": 4.89642567018684e-06,
+      "loss": 2.0866,
+      "step": 339
+    },
+    {
+      "epoch": 0.3726027397260274,
+      "grad_norm": 2.6379668712615967,
+      "learning_rate": 4.894394800974818e-06,
+      "loss": 2.1552,
+      "step": 340
+    },
+    {
+      "epoch": 0.3726027397260274,
+      "eval_loss": 2.196237802505493,
+      "eval_runtime": 36.8953,
+      "eval_samples_per_second": 27.104,
+      "eval_steps_per_second": 1.138,
+      "step": 340
+    },
+    {
+      "epoch": 0.3736986301369863,
+      "grad_norm": 2.417145252227783,
+      "learning_rate": 4.892363931762794e-06,
+      "loss": 2.0024,
+      "step": 341
+    },
+    {
+      "epoch": 0.3747945205479452,
+      "grad_norm": 2.234590768814087,
+      "learning_rate": 4.8903330625507725e-06,
+      "loss": 2.0837,
+      "step": 342
+    },
+    {
+      "epoch": 0.37589041095890413,
+      "grad_norm": 2.404266595840454,
+      "learning_rate": 4.888302193338749e-06,
+      "loss": 2.018,
+      "step": 343
+    },
+    {
+      "epoch": 0.376986301369863,
+      "grad_norm": 2.364802837371826,
+      "learning_rate": 4.886271324126727e-06,
+      "loss": 1.994,
+      "step": 344
+    },
+    {
+      "epoch": 0.3780821917808219,
+      "grad_norm": 2.2262251377105713,
+      "learning_rate": 4.8842404549147034e-06,
+      "loss": 2.0929,
+      "step": 345
+    },
+    {
+      "epoch": 0.37917808219178084,
+      "grad_norm": 2.4854862689971924,
+      "learning_rate": 4.8822095857026816e-06,
+      "loss": 2.0801,
+      "step": 346
+    },
+    {
+      "epoch": 0.3802739726027397,
+      "grad_norm": 2.3861470222473145,
+      "learning_rate": 4.880178716490658e-06,
+      "loss": 2.0496,
+      "step": 347
+    },
+    {
+      "epoch": 0.38136986301369863,
+      "grad_norm": 2.372326612472534,
+      "learning_rate": 4.878147847278636e-06,
+      "loss": 1.9939,
+      "step": 348
+    },
+    {
+      "epoch": 0.38246575342465755,
+      "grad_norm": 2.3593149185180664,
+      "learning_rate": 4.8761169780666125e-06,
+      "loss": 2.0108,
+      "step": 349
+    },
+    {
+      "epoch": 0.3835616438356164,
+      "grad_norm": 2.1802141666412354,
+      "learning_rate": 4.874086108854591e-06,
+      "loss": 1.983,
+      "step": 350
+    },
+    {
+      "epoch": 0.3835616438356164,
+      "eval_loss": 2.1947567462921143,
+      "eval_runtime": 36.8798,
+      "eval_samples_per_second": 27.115,
+      "eval_steps_per_second": 1.139,
+      "step": 350
+    },
+    {
+      "epoch": 0.38465753424657534,
+      "grad_norm": 2.47067928314209,
+      "learning_rate": 4.872055239642567e-06,
+      "loss": 2.0002,
+      "step": 351
+    },
+    {
+      "epoch": 0.38575342465753426,
+      "grad_norm": 2.359609365463257,
+      "learning_rate": 4.870024370430544e-06,
+      "loss": 1.9799,
+      "step": 352
+    },
+    {
+      "epoch": 0.3868493150684931,
+      "grad_norm": 2.3711702823638916,
+      "learning_rate": 4.867993501218522e-06,
+      "loss": 2.0341,
+      "step": 353
+    },
+    {
+      "epoch": 0.38794520547945205,
+      "grad_norm": 2.246647834777832,
+      "learning_rate": 4.865962632006499e-06,
+      "loss": 2.0687,
+      "step": 354
+    },
+    {
+      "epoch": 0.38904109589041097,
+      "grad_norm": 2.3031938076019287,
+      "learning_rate": 4.863931762794476e-06,
+      "loss": 1.9994,
+      "step": 355
+    },
+    {
+      "epoch": 0.39013698630136984,
+      "grad_norm": 2.492159605026245,
+      "learning_rate": 4.8619008935824535e-06,
+      "loss": 2.0195,
+      "step": 356
+    },
+    {
+      "epoch": 0.39123287671232876,
+      "grad_norm": 2.2878570556640625,
+      "learning_rate": 4.859870024370431e-06,
+      "loss": 2.0589,
+      "step": 357
+    },
+    {
+      "epoch": 0.3923287671232877,
+      "grad_norm": 2.444096088409424,
+      "learning_rate": 4.857839155158408e-06,
+      "loss": 2.0796,
+      "step": 358
+    },
+    {
+      "epoch": 0.3934246575342466,
+      "grad_norm": 2.463857650756836,
+      "learning_rate": 4.855808285946385e-06,
+      "loss": 2.0824,
+      "step": 359
+    },
+    {
+      "epoch": 0.39452054794520547,
+      "grad_norm": 2.0893168449401855,
+      "learning_rate": 4.8537774167343626e-06,
+      "loss": 2.0651,
+      "step": 360
+    },
+    {
+      "epoch": 0.39452054794520547,
+      "eval_loss": 2.188523054122925,
+      "eval_runtime": 36.8679,
+      "eval_samples_per_second": 27.124,
+      "eval_steps_per_second": 1.139,
+      "step": 360
+    },
+    {
+      "epoch": 0.3956164383561644,
+      "grad_norm": 2.602850914001465,
+      "learning_rate": 4.85174654752234e-06,
+      "loss": 2.0507,
+      "step": 361
+    },
+    {
+      "epoch": 0.3967123287671233,
+      "grad_norm": 2.4185140132904053,
+      "learning_rate": 4.849715678310317e-06,
+      "loss": 2.0386,
+      "step": 362
+    },
+    {
+      "epoch": 0.3978082191780822,
+      "grad_norm": 2.1492245197296143,
+      "learning_rate": 4.847684809098294e-06,
+      "loss": 2.0376,
+      "step": 363
+    },
+    {
+      "epoch": 0.3989041095890411,
+      "grad_norm": 2.5710153579711914,
+      "learning_rate": 4.845653939886272e-06,
+      "loss": 2.0383,
+      "step": 364
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.4986608028411865,
+      "learning_rate": 4.843623070674249e-06,
+      "loss": 2.0464,
+      "step": 365
+    },
+    {
+      "epoch": 0.4010958904109589,
+      "grad_norm": 2.260388135910034,
+      "learning_rate": 4.841592201462226e-06,
+      "loss": 2.0217,
+      "step": 366
+    },
+    {
+      "epoch": 0.4021917808219178,
+      "grad_norm": 2.6500394344329834,
+      "learning_rate": 4.8395613322502035e-06,
+      "loss": 2.0777,
+      "step": 367
+    },
+    {
+      "epoch": 0.40328767123287673,
+      "grad_norm": 2.159274101257324,
+      "learning_rate": 4.837530463038181e-06,
+      "loss": 1.9747,
+      "step": 368
+    },
+    {
+      "epoch": 0.4043835616438356,
+      "grad_norm": 2.4623043537139893,
+      "learning_rate": 4.835499593826158e-06,
+      "loss": 2.039,
+      "step": 369
+    },
+    {
+      "epoch": 0.4054794520547945,
+      "grad_norm": 2.2265007495880127,
+      "learning_rate": 4.833468724614135e-06,
+      "loss": 2.0618,
+      "step": 370
+    },
+    {
+      "epoch": 0.4054794520547945,
+      "eval_loss": 2.184051275253296,
+      "eval_runtime": 36.9284,
+      "eval_samples_per_second": 27.079,
+      "eval_steps_per_second": 1.137,
+      "step": 370
+    },
+    {
+      "epoch": 0.40657534246575344,
+      "grad_norm": 2.300276517868042,
+      "learning_rate": 4.831437855402113e-06,
+      "loss": 2.0382,
+      "step": 371
+    },
+    {
+      "epoch": 0.4076712328767123,
+      "grad_norm": 2.341156482696533,
+      "learning_rate": 4.82940698619009e-06,
+      "loss": 1.9361,
+      "step": 372
+    },
+    {
+      "epoch": 0.40876712328767123,
+      "grad_norm": 2.4196999073028564,
+      "learning_rate": 4.827376116978067e-06,
+      "loss": 2.0087,
+      "step": 373
+    },
+    {
+      "epoch": 0.40986301369863015,
+      "grad_norm": 2.4650514125823975,
+      "learning_rate": 4.8253452477660444e-06,
+      "loss": 2.0687,
+      "step": 374
+    },
+    {
+      "epoch": 0.410958904109589,
+      "grad_norm": 2.393739700317383,
+      "learning_rate": 4.823314378554022e-06,
+      "loss": 2.0421,
+      "step": 375
+    },
+    {
+      "epoch": 0.41205479452054794,
+      "grad_norm": 2.2436327934265137,
+      "learning_rate": 4.821283509341998e-06,
+      "loss": 2.0358,
+      "step": 376
+    },
+    {
+      "epoch": 0.41315068493150686,
+      "grad_norm": 2.1801650524139404,
+      "learning_rate": 4.819252640129976e-06,
+      "loss": 2.0081,
+      "step": 377
+    },
+    {
+      "epoch": 0.41424657534246573,
+      "grad_norm": 2.300882339477539,
+      "learning_rate": 4.817221770917953e-06,
+      "loss": 2.0321,
+      "step": 378
+    },
+    {
+      "epoch": 0.41534246575342465,
+      "grad_norm": 2.2136058807373047,
+      "learning_rate": 4.815190901705931e-06,
+      "loss": 1.9999,
+      "step": 379
+    },
+    {
+      "epoch": 0.41643835616438357,
+      "grad_norm": 2.168034076690674,
+      "learning_rate": 4.813160032493907e-06,
+      "loss": 2.0336,
+      "step": 380
+    },
+    {
+      "epoch": 0.41643835616438357,
+      "eval_loss": 2.1770179271698,
+      "eval_runtime": 37.0015,
+      "eval_samples_per_second": 27.026,
+      "eval_steps_per_second": 1.135,
+      "step": 380
+    },
+    {
+      "epoch": 0.41753424657534244,
+      "grad_norm": 2.1604163646698,
+      "learning_rate": 4.811129163281885e-06,
+      "loss": 2.0212,
+      "step": 381
+    },
+    {
+      "epoch": 0.41863013698630136,
+      "grad_norm": 2.23923659324646,
+      "learning_rate": 4.809098294069862e-06,
+      "loss": 2.0495,
+      "step": 382
+    },
+    {
+      "epoch": 0.4197260273972603,
+      "grad_norm": 2.149949312210083,
+      "learning_rate": 4.80706742485784e-06,
+      "loss": 2.041,
+      "step": 383
+    },
+    {
+      "epoch": 0.4208219178082192,
+      "grad_norm": 2.49120831489563,
+      "learning_rate": 4.805036555645816e-06,
+      "loss": 2.0931,
+      "step": 384
+    },
+    {
+      "epoch": 0.42191780821917807,
+      "grad_norm": 2.2123563289642334,
+      "learning_rate": 4.8030056864337945e-06,
+      "loss": 2.0087,
+      "step": 385
+    },
+    {
+      "epoch": 0.423013698630137,
+      "grad_norm": 2.4235594272613525,
+      "learning_rate": 4.800974817221771e-06,
+      "loss": 2.0656,
+      "step": 386
+    },
+    {
+      "epoch": 0.4241095890410959,
+      "grad_norm": 2.158252239227295,
+      "learning_rate": 4.798943948009749e-06,
+      "loss": 2.0181,
+      "step": 387
+    },
+    {
+      "epoch": 0.4252054794520548,
+      "grad_norm": 2.2412970066070557,
+      "learning_rate": 4.7969130787977254e-06,
+      "loss": 2.0323,
+      "step": 388
+    },
+    {
+      "epoch": 0.4263013698630137,
+      "grad_norm": 2.2520956993103027,
+      "learning_rate": 4.794882209585703e-06,
+      "loss": 2.026,
+      "step": 389
+    },
+    {
+      "epoch": 0.4273972602739726,
+      "grad_norm": 2.168954849243164,
+      "learning_rate": 4.79285134037368e-06,
+      "loss": 2.0459,
+      "step": 390
+    },
+    {
+      "epoch": 0.4273972602739726,
+      "eval_loss": 2.1705472469329834,
+      "eval_runtime": 37.0913,
+      "eval_samples_per_second": 26.96,
+      "eval_steps_per_second": 1.132,
+      "step": 390
+    },
+    {
+      "epoch": 0.4284931506849315,
+      "grad_norm": 2.1397407054901123,
+      "learning_rate": 4.790820471161657e-06,
+      "loss": 1.9995,
+      "step": 391
+    },
+    {
+      "epoch": 0.4295890410958904,
+      "grad_norm": 2.1747751235961914,
+      "learning_rate": 4.7887896019496345e-06,
+      "loss": 2.0088,
+      "step": 392
+    },
+    {
+      "epoch": 0.43068493150684933,
+      "grad_norm": 2.2860991954803467,
+      "learning_rate": 4.786758732737612e-06,
+      "loss": 2.0134,
+      "step": 393
+    },
+    {
+      "epoch": 0.4317808219178082,
+      "grad_norm": 2.2133519649505615,
+      "learning_rate": 4.784727863525589e-06,
+      "loss": 2.0586,
+      "step": 394
+    },
+    {
+      "epoch": 0.4328767123287671,
+      "grad_norm": 2.2147040367126465,
+      "learning_rate": 4.782696994313566e-06,
+      "loss": 1.9748,
+      "step": 395
+    },
+    {
+      "epoch": 0.43397260273972604,
+      "grad_norm": 2.343418836593628,
+      "learning_rate": 4.780666125101544e-06,
+      "loss": 2.0223,
+      "step": 396
+    },
+    {
+      "epoch": 0.4350684931506849,
+      "grad_norm": 2.198935031890869,
+      "learning_rate": 4.778635255889521e-06,
+      "loss": 1.9627,
+      "step": 397
+    },
+    {
+      "epoch": 0.43616438356164383,
+      "grad_norm": 2.1314704418182373,
+      "learning_rate": 4.776604386677498e-06,
+      "loss": 1.9938,
+      "step": 398
+    },
+    {
+      "epoch": 0.43726027397260275,
+      "grad_norm": 2.3005123138427734,
+      "learning_rate": 4.7745735174654755e-06,
+      "loss": 2.0029,
+      "step": 399
+    },
+    {
+      "epoch": 0.4383561643835616,
+      "grad_norm": 2.2826664447784424,
+      "learning_rate": 4.772542648253453e-06,
+      "loss": 2.0585,
+      "step": 400
+    },
+    {
+      "epoch": 0.4383561643835616,
+      "eval_loss": 2.1691415309906006,
+      "eval_runtime": 37.0852,
+      "eval_samples_per_second": 26.965,
+      "eval_steps_per_second": 1.133,
+      "step": 400
+    },
+    {
+      "epoch": 0.43945205479452054,
+      "grad_norm": 2.0547878742218018,
+      "learning_rate": 4.77051177904143e-06,
+      "loss": 2.0582,
+      "step": 401
+    },
+    {
+      "epoch": 0.44054794520547946,
+      "grad_norm": 2.200042486190796,
+      "learning_rate": 4.768480909829407e-06,
+      "loss": 1.9936,
+      "step": 402
+    },
+    {
+      "epoch": 0.44164383561643833,
+      "grad_norm": 2.354637861251831,
+      "learning_rate": 4.766450040617385e-06,
+      "loss": 2.0327,
+      "step": 403
+    },
+    {
+      "epoch": 0.44273972602739725,
+      "grad_norm": 2.2332348823547363,
+      "learning_rate": 4.764419171405362e-06,
+      "loss": 2.0052,
+      "step": 404
+    },
+    {
+      "epoch": 0.4438356164383562,
+      "grad_norm": 2.288060426712036,
+      "learning_rate": 4.762388302193339e-06,
+      "loss": 2.0472,
+      "step": 405
+    },
+    {
+      "epoch": 0.4449315068493151,
+      "grad_norm": 2.1368887424468994,
+      "learning_rate": 4.760357432981316e-06,
+      "loss": 2.0305,
+      "step": 406
+    },
+    {
+      "epoch": 0.44602739726027396,
+      "grad_norm": 2.362630844116211,
+      "learning_rate": 4.758326563769294e-06,
+      "loss": 2.002,
+      "step": 407
+    },
+    {
+      "epoch": 0.4471232876712329,
+      "grad_norm": 2.283419370651245,
+      "learning_rate": 4.756295694557271e-06,
+      "loss": 2.0271,
+      "step": 408
+    },
+    {
+      "epoch": 0.4482191780821918,
+      "grad_norm": 2.0400819778442383,
+      "learning_rate": 4.754264825345248e-06,
+      "loss": 2.0256,
+      "step": 409
+    },
+    {
+      "epoch": 0.44931506849315067,
+      "grad_norm": 2.588416814804077,
+      "learning_rate": 4.7522339561332255e-06,
+      "loss": 1.9756,
+      "step": 410
+    },
+    {
+      "epoch": 0.44931506849315067,
+      "eval_loss": 2.17041015625,
+      "eval_runtime": 36.958,
+      "eval_samples_per_second": 27.058,
+      "eval_steps_per_second": 1.136,
+      "step": 410
+    },
+    {
+      "epoch": 0.4504109589041096,
+      "grad_norm": 2.2397494316101074,
+      "learning_rate": 4.750203086921203e-06,
+      "loss": 1.9876,
+      "step": 411
+    },
+    {
+      "epoch": 0.4515068493150685,
+      "grad_norm": 2.2085509300231934,
+      "learning_rate": 4.74817221770918e-06,
+      "loss": 1.9464,
+      "step": 412
+    },
+    {
+      "epoch": 0.4526027397260274,
+      "grad_norm": 2.140974760055542,
+      "learning_rate": 4.7461413484971565e-06,
+      "loss": 2.0179,
+      "step": 413
+    },
+    {
+      "epoch": 0.4536986301369863,
+      "grad_norm": 2.124366044998169,
+      "learning_rate": 4.744110479285135e-06,
+      "loss": 2.0732,
+      "step": 414
+    },
+    {
+      "epoch": 0.4547945205479452,
+      "grad_norm": 2.2463996410369873,
+      "learning_rate": 4.742079610073111e-06,
+      "loss": 1.9616,
+      "step": 415
+    },
+    {
+      "epoch": 0.4558904109589041,
+      "grad_norm": 2.1249961853027344,
+      "learning_rate": 4.740048740861089e-06,
+      "loss": 1.9889,
+      "step": 416
+    },
+    {
+      "epoch": 0.456986301369863,
+      "grad_norm": 2.25842022895813,
+      "learning_rate": 4.738017871649066e-06,
+      "loss": 2.0033,
+      "step": 417
+    },
+    {
+      "epoch": 0.45808219178082193,
+      "grad_norm": 2.271935224533081,
+      "learning_rate": 4.735987002437044e-06,
+      "loss": 2.0148,
+      "step": 418
+    },
+    {
+      "epoch": 0.4591780821917808,
+      "grad_norm": 2.4797163009643555,
+      "learning_rate": 4.73395613322502e-06,
+      "loss": 2.0736,
+      "step": 419
+    },
+    {
+      "epoch": 0.4602739726027397,
+      "grad_norm": 2.1901395320892334,
+      "learning_rate": 4.731925264012998e-06,
+      "loss": 2.0763,
+      "step": 420
+    },
+    {
+      "epoch": 0.4602739726027397,
+      "eval_loss": 2.161660671234131,
+      "eval_runtime": 36.9099,
+      "eval_samples_per_second": 27.093,
+      "eval_steps_per_second": 1.138,
+      "step": 420
+    },
+    {
+      "epoch": 0.46136986301369864,
+      "grad_norm": 2.083536386489868,
+      "learning_rate": 4.729894394800975e-06,
+      "loss": 2.0299,
+      "step": 421
+    },
+    {
+      "epoch": 0.4624657534246575,
+      "grad_norm": 2.1118173599243164,
+      "learning_rate": 4.727863525588953e-06,
+      "loss": 1.9742,
+      "step": 422
+    },
+    {
+      "epoch": 0.46356164383561643,
+      "grad_norm": 2.195802927017212,
+      "learning_rate": 4.725832656376929e-06,
+      "loss": 1.9318,
+      "step": 423
+    },
+    {
+      "epoch": 0.46465753424657535,
+      "grad_norm": 2.318098545074463,
+      "learning_rate": 4.723801787164907e-06,
+      "loss": 1.9875,
+      "step": 424
+    },
+    {
+      "epoch": 0.4657534246575342,
+      "grad_norm": 2.1698496341705322,
+      "learning_rate": 4.721770917952885e-06,
+      "loss": 2.031,
+      "step": 425
+    },
+    {
+      "epoch": 0.46684931506849314,
+      "grad_norm": 2.283935785293579,
+      "learning_rate": 4.719740048740861e-06,
+      "loss": 1.9766,
+      "step": 426
+    },
+    {
+      "epoch": 0.46794520547945206,
+      "grad_norm": 2.4180757999420166,
+      "learning_rate": 4.717709179528839e-06,
+      "loss": 1.9976,
+      "step": 427
+    },
+    {
+      "epoch": 0.469041095890411,
+      "grad_norm": 2.2689030170440674,
+      "learning_rate": 4.715678310316816e-06,
+      "loss": 2.03,
+      "step": 428
+    },
+    {
+      "epoch": 0.47013698630136985,
+      "grad_norm": 2.423708200454712,
+      "learning_rate": 4.713647441104794e-06,
+      "loss": 2.0202,
+      "step": 429
+    },
+    {
+      "epoch": 0.4712328767123288,
+      "grad_norm": 2.184705972671509,
+      "learning_rate": 4.71161657189277e-06,
+      "loss": 1.9655,
+      "step": 430
+    },
+    {
+      "epoch": 0.4712328767123288,
+      "eval_loss": 2.1548638343811035,
+      "eval_runtime": 36.9426,
+      "eval_samples_per_second": 27.069,
+      "eval_steps_per_second": 1.137,
+      "step": 430
+    },
+    {
+      "epoch": 0.4723287671232877,
+      "grad_norm": 2.295851945877075,
+      "learning_rate": 4.709585702680748e-06,
+      "loss": 1.9561,
+      "step": 431
+    },
+    {
+      "epoch": 0.47342465753424656,
+      "grad_norm": 2.1762046813964844,
+      "learning_rate": 4.707554833468725e-06,
+      "loss": 2.0341,
+      "step": 432
+    },
+    {
+      "epoch": 0.4745205479452055,
+      "grad_norm": 2.4684343338012695,
+      "learning_rate": 4.705523964256703e-06,
+      "loss": 2.0898,
+      "step": 433
+    },
+    {
+      "epoch": 0.4756164383561644,
+      "grad_norm": 2.293318271636963,
+      "learning_rate": 4.703493095044679e-06,
+      "loss": 1.9794,
+      "step": 434
+    },
+    {
+      "epoch": 0.4767123287671233,
+      "grad_norm": 2.1884210109710693,
+      "learning_rate": 4.701462225832657e-06,
+      "loss": 1.9979,
+      "step": 435
+    },
+    {
+      "epoch": 0.4778082191780822,
+      "grad_norm": 2.412504196166992,
+      "learning_rate": 4.699431356620634e-06,
+      "loss": 1.9237,
+      "step": 436
+    },
+    {
+      "epoch": 0.4789041095890411,
+      "grad_norm": 2.192824602127075,
+      "learning_rate": 4.697400487408612e-06,
+      "loss": 2.0756,
+      "step": 437
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 2.368100643157959,
+      "learning_rate": 4.695369618196588e-06,
+      "loss": 2.028,
+      "step": 438
+    },
+    {
+      "epoch": 0.4810958904109589,
+      "grad_norm": 2.218454122543335,
+      "learning_rate": 4.693338748984566e-06,
+      "loss": 2.0049,
+      "step": 439
+    },
+    {
+      "epoch": 0.4821917808219178,
+      "grad_norm": 2.356168031692505,
+      "learning_rate": 4.691307879772543e-06,
+      "loss": 1.9958,
+      "step": 440
+    },
+    {
+      "epoch": 0.4821917808219178,
+      "eval_loss": 2.1540727615356445,
+      "eval_runtime": 36.9557,
+      "eval_samples_per_second": 27.059,
+      "eval_steps_per_second": 1.136,
+      "step": 440
+    },
+    {
+      "epoch": 0.4832876712328767,
+      "grad_norm": 2.09956693649292,
+      "learning_rate": 4.68927701056052e-06,
+      "loss": 1.9388,
+      "step": 441
+    },
+    {
+      "epoch": 0.4843835616438356,
+      "grad_norm": 2.5746614933013916,
+      "learning_rate": 4.6872461413484975e-06,
+      "loss": 2.0425,
+      "step": 442
+    },
+    {
+      "epoch": 0.48547945205479454,
+      "grad_norm": 2.2064783573150635,
+      "learning_rate": 4.685215272136475e-06,
+      "loss": 2.0082,
+      "step": 443
+    },
+    {
+      "epoch": 0.4865753424657534,
+      "grad_norm": 2.159029006958008,
+      "learning_rate": 4.683184402924452e-06,
+      "loss": 2.0055,
+      "step": 444
+    },
+    {
+      "epoch": 0.4876712328767123,
+      "grad_norm": 2.5222008228302,
+      "learning_rate": 4.681153533712429e-06,
+      "loss": 2.0007,
+      "step": 445
+    },
+    {
+      "epoch": 0.48876712328767125,
+      "grad_norm": 2.163487672805786,
+      "learning_rate": 4.679122664500407e-06,
+      "loss": 2.0719,
+      "step": 446
+    },
+    {
+      "epoch": 0.4898630136986301,
+      "grad_norm": 2.3383803367614746,
+      "learning_rate": 4.677091795288384e-06,
+      "loss": 1.9623,
+      "step": 447
+    },
+    {
+      "epoch": 0.49095890410958903,
+      "grad_norm": 2.1056675910949707,
+      "learning_rate": 4.675060926076361e-06,
+      "loss": 1.9443,
+      "step": 448
+    },
+    {
+      "epoch": 0.49205479452054796,
+      "grad_norm": 2.145270824432373,
+      "learning_rate": 4.673030056864338e-06,
+      "loss": 2.0469,
+      "step": 449
+    },
+    {
+      "epoch": 0.4931506849315068,
+      "grad_norm": 2.578706741333008,
+      "learning_rate": 4.670999187652316e-06,
+      "loss": 1.9569,
+      "step": 450
+    },
+    {
+      "epoch": 0.4931506849315068,
+      "eval_loss": 2.1527743339538574,
+      "eval_runtime": 36.8961,
+      "eval_samples_per_second": 27.103,
+      "eval_steps_per_second": 1.138,
+      "step": 450
+    },
+    {
+      "epoch": 0.49424657534246574,
+      "grad_norm": 2.1328542232513428,
+      "learning_rate": 4.668968318440293e-06,
+      "loss": 1.9559,
+      "step": 451
+    },
+    {
+      "epoch": 0.49534246575342467,
+      "grad_norm": 2.148820638656616,
+      "learning_rate": 4.66693744922827e-06,
+      "loss": 1.9545,
+      "step": 452
+    },
+    {
+      "epoch": 0.4964383561643836,
+      "grad_norm": 2.250340461730957,
+      "learning_rate": 4.6649065800162475e-06,
+      "loss": 2.0358,
+      "step": 453
+    },
+    {
+      "epoch": 0.49753424657534245,
+      "grad_norm": 2.452397584915161,
+      "learning_rate": 4.662875710804225e-06,
+      "loss": 1.9294,
+      "step": 454
+    },
+    {
+      "epoch": 0.4986301369863014,
+      "grad_norm": 2.140615701675415,
+      "learning_rate": 4.660844841592202e-06,
+      "loss": 2.0084,
+      "step": 455
+    },
+    {
+      "epoch": 0.4997260273972603,
+      "grad_norm": 2.176806926727295,
+      "learning_rate": 4.658813972380179e-06,
+      "loss": 1.9174,
+      "step": 456
+    },
+    {
+      "epoch": 0.5008219178082192,
+      "grad_norm": 2.349031925201416,
+      "learning_rate": 4.656783103168157e-06,
+      "loss": 2.0377,
+      "step": 457
+    },
+    {
+      "epoch": 0.5019178082191781,
+      "grad_norm": 2.436889171600342,
+      "learning_rate": 4.654752233956134e-06,
+      "loss": 1.9133,
+      "step": 458
+    },
+    {
+      "epoch": 0.503013698630137,
+      "grad_norm": 2.116610288619995,
+      "learning_rate": 4.652721364744111e-06,
+      "loss": 1.9993,
+      "step": 459
+    },
+    {
+      "epoch": 0.5041095890410959,
+      "grad_norm": 2.186915636062622,
+      "learning_rate": 4.6506904955320885e-06,
+      "loss": 1.9744,
+      "step": 460
+    },
+    {
+      "epoch": 0.5041095890410959,
+      "eval_loss": 2.1580278873443604,
+      "eval_runtime": 36.9097,
+      "eval_samples_per_second": 27.093,
+      "eval_steps_per_second": 1.138,
+      "step": 460
+    },
+    {
+      "epoch": 0.5052054794520547,
+      "grad_norm": 2.2009212970733643,
+      "learning_rate": 4.648659626320066e-06,
+      "loss": 2.0173,
+      "step": 461
+    },
+    {
+      "epoch": 0.5063013698630137,
+      "grad_norm": 2.117286443710327,
+      "learning_rate": 4.646628757108043e-06,
+      "loss": 2.0238,
+      "step": 462
+    },
+    {
+      "epoch": 0.5073972602739726,
+      "grad_norm": 2.238790273666382,
+      "learning_rate": 4.6445978878960194e-06,
+      "loss": 1.9885,
+      "step": 463
+    },
+    {
+      "epoch": 0.5084931506849315,
+      "grad_norm": 2.3493096828460693,
+      "learning_rate": 4.6425670186839976e-06,
+      "loss": 1.9593,
+      "step": 464
+    },
+    {
+      "epoch": 0.5095890410958904,
+      "grad_norm": 2.3276610374450684,
+      "learning_rate": 4.640536149471974e-06,
+      "loss": 1.9993,
+      "step": 465
+    },
+    {
+      "epoch": 0.5106849315068493,
+      "grad_norm": 2.3671715259552,
+      "learning_rate": 4.638505280259952e-06,
+      "loss": 2.0256,
+      "step": 466
+    },
+    {
+      "epoch": 0.5117808219178083,
+      "grad_norm": 2.251803159713745,
+      "learning_rate": 4.6364744110479285e-06,
+      "loss": 2.0436,
+      "step": 467
+    },
+    {
+      "epoch": 0.5128767123287671,
+      "grad_norm": 2.3023312091827393,
+      "learning_rate": 4.634443541835907e-06,
+      "loss": 1.9968,
+      "step": 468
+    },
+    {
+      "epoch": 0.513972602739726,
+      "grad_norm": 2.3660333156585693,
+      "learning_rate": 4.632412672623883e-06,
+      "loss": 2.0492,
+      "step": 469
+    },
+    {
+      "epoch": 0.5150684931506849,
+      "grad_norm": 2.262179136276245,
+      "learning_rate": 4.630381803411861e-06,
+      "loss": 1.9698,
+      "step": 470
+    },
+    {
+      "epoch": 0.5150684931506849,
+      "eval_loss": 2.1516528129577637,
+      "eval_runtime": 36.7956,
+      "eval_samples_per_second": 27.177,
+      "eval_steps_per_second": 1.141,
+      "step": 470
+    },
+    {
+      "epoch": 0.5161643835616438,
+      "grad_norm": 2.450770616531372,
+      "learning_rate": 4.628350934199838e-06,
+      "loss": 1.8799,
+      "step": 471
+    },
+    {
+      "epoch": 0.5172602739726028,
+      "grad_norm": 2.187458038330078,
+      "learning_rate": 4.626320064987816e-06,
+      "loss": 1.9829,
+      "step": 472
+    },
+    {
+      "epoch": 0.5183561643835617,
+      "grad_norm": 2.1726722717285156,
+      "learning_rate": 4.624289195775792e-06,
+      "loss": 2.012,
+      "step": 473
+    },
+    {
+      "epoch": 0.5194520547945205,
+      "grad_norm": 2.4177136421203613,
+      "learning_rate": 4.62225832656377e-06,
+      "loss": 1.9796,
+      "step": 474
+    },
+    {
+      "epoch": 0.5205479452054794,
+      "grad_norm": 2.2353951930999756,
+      "learning_rate": 4.620227457351747e-06,
+      "loss": 1.99,
+      "step": 475
+    },
+    {
+      "epoch": 0.5216438356164383,
+      "grad_norm": 2.369591236114502,
+      "learning_rate": 4.618196588139724e-06,
+      "loss": 1.9847,
+      "step": 476
+    },
+    {
+      "epoch": 0.5227397260273973,
+      "grad_norm": 2.546703577041626,
+      "learning_rate": 4.616165718927701e-06,
+      "loss": 2.0047,
+      "step": 477
+    },
+    {
+      "epoch": 0.5238356164383562,
+      "grad_norm": 2.3509185314178467,
+      "learning_rate": 4.6141348497156786e-06,
+      "loss": 1.9589,
+      "step": 478
+    },
+    {
+      "epoch": 0.5249315068493151,
+      "grad_norm": 2.3063488006591797,
+      "learning_rate": 4.612103980503656e-06,
+      "loss": 2.045,
+      "step": 479
+    },
+    {
+      "epoch": 0.5260273972602739,
+      "grad_norm": 2.285572052001953,
+      "learning_rate": 4.610073111291633e-06,
+      "loss": 2.0218,
+      "step": 480
+    },
+    {
+      "epoch": 0.5260273972602739,
+      "eval_loss": 2.1472301483154297,
+      "eval_runtime": 36.7689,
+      "eval_samples_per_second": 27.197,
+      "eval_steps_per_second": 1.142,
+      "step": 480
+    },
+    {
+      "epoch": 0.5271232876712328,
+      "grad_norm": 2.22987961769104,
+      "learning_rate": 4.60804224207961e-06,
+      "loss": 1.9955,
+      "step": 481
+    },
+    {
+      "epoch": 0.5282191780821918,
+      "grad_norm": 2.200918197631836,
+      "learning_rate": 4.606011372867588e-06,
+      "loss": 1.9768,
+      "step": 482
+    },
+    {
+      "epoch": 0.5293150684931507,
+      "grad_norm": 2.2441930770874023,
+      "learning_rate": 4.603980503655565e-06,
+      "loss": 2.0088,
+      "step": 483
+    },
+    {
+      "epoch": 0.5304109589041096,
+      "grad_norm": 2.380383014678955,
+      "learning_rate": 4.601949634443542e-06,
+      "loss": 1.9853,
+      "step": 484
+    },
+    {
+      "epoch": 0.5315068493150685,
+      "grad_norm": 2.317995071411133,
+      "learning_rate": 4.5999187652315195e-06,
+      "loss": 1.9183,
+      "step": 485
+    },
+    {
+      "epoch": 0.5326027397260275,
+      "grad_norm": 2.2462668418884277,
+      "learning_rate": 4.597887896019497e-06,
+      "loss": 1.9201,
+      "step": 486
+    },
+    {
+      "epoch": 0.5336986301369863,
+      "grad_norm": 2.1482582092285156,
+      "learning_rate": 4.595857026807474e-06,
+      "loss": 1.9722,
+      "step": 487
+    },
+    {
+      "epoch": 0.5347945205479452,
+      "grad_norm": 2.079831123352051,
+      "learning_rate": 4.593826157595451e-06,
+      "loss": 1.977,
+      "step": 488
+    },
+    {
+      "epoch": 0.5358904109589041,
+      "grad_norm": 2.2169578075408936,
+      "learning_rate": 4.591795288383429e-06,
+      "loss": 1.9615,
+      "step": 489
+    },
+    {
+      "epoch": 0.536986301369863,
+      "grad_norm": 2.087846279144287,
+      "learning_rate": 4.589764419171406e-06,
+      "loss": 2.0402,
+      "step": 490
+    },
+    {
+      "epoch": 0.536986301369863,
+      "eval_loss": 2.1416702270507812,
+      "eval_runtime": 37.1191,
+      "eval_samples_per_second": 26.94,
+      "eval_steps_per_second": 1.131,
+      "step": 490
+    },
+    {
+      "epoch": 0.538082191780822,
+      "grad_norm": 2.2161977291107178,
+      "learning_rate": 4.587733549959383e-06,
+      "loss": 1.9959,
+      "step": 491
+    },
+    {
+      "epoch": 0.5391780821917809,
+      "grad_norm": 2.19530987739563,
+      "learning_rate": 4.5857026807473604e-06,
+      "loss": 1.961,
+      "step": 492
+    },
+    {
+      "epoch": 0.5402739726027397,
+      "grad_norm": 2.1150615215301514,
+      "learning_rate": 4.583671811535338e-06,
+      "loss": 1.946,
+      "step": 493
+    },
+    {
+      "epoch": 0.5413698630136986,
+      "grad_norm": 2.186352491378784,
+      "learning_rate": 4.581640942323315e-06,
+      "loss": 1.9137,
+      "step": 494
+    },
+    {
+      "epoch": 0.5424657534246575,
+      "grad_norm": 2.233370780944824,
+      "learning_rate": 4.579610073111292e-06,
+      "loss": 2.021,
+      "step": 495
+    },
+    {
+      "epoch": 0.5435616438356164,
+      "grad_norm": 2.1326382160186768,
+      "learning_rate": 4.5775792038992695e-06,
+      "loss": 2.0242,
+      "step": 496
+    },
+    {
+      "epoch": 0.5446575342465754,
+      "grad_norm": 2.536837577819824,
+      "learning_rate": 4.575548334687247e-06,
+      "loss": 2.0185,
+      "step": 497
+    },
+    {
+      "epoch": 0.5457534246575343,
+      "grad_norm": 2.193598747253418,
+      "learning_rate": 4.573517465475224e-06,
+      "loss": 2.0533,
+      "step": 498
+    },
+    {
+      "epoch": 0.5468493150684931,
+      "grad_norm": 2.1346311569213867,
+      "learning_rate": 4.571486596263201e-06,
+      "loss": 1.9564,
+      "step": 499
+    },
+    {
+      "epoch": 0.547945205479452,
+      "grad_norm": 2.4294722080230713,
+      "learning_rate": 4.569455727051178e-06,
+      "loss": 2.0003,
+      "step": 500
+    },
+    {
+      "epoch": 0.547945205479452,
+      "eval_loss": 2.1430764198303223,
+      "eval_runtime": 37.0591,
+      "eval_samples_per_second": 26.984,
+      "eval_steps_per_second": 1.133,
+      "step": 500
+    },
+    {
+      "epoch": 0.549041095890411,
+      "grad_norm": 2.2394182682037354,
+      "learning_rate": 4.567424857839156e-06,
+      "loss": 2.0055,
+      "step": 501
+    },
+    {
+      "epoch": 0.5501369863013699,
+      "grad_norm": 2.2543067932128906,
+      "learning_rate": 4.565393988627132e-06,
+      "loss": 1.9837,
+      "step": 502
+    },
+    {
+      "epoch": 0.5512328767123288,
+      "grad_norm": 2.2805464267730713,
+      "learning_rate": 4.5633631194151105e-06,
+      "loss": 2.0495,
+      "step": 503
+    },
+    {
+      "epoch": 0.5523287671232877,
+      "grad_norm": 2.2838146686553955,
+      "learning_rate": 4.561332250203087e-06,
+      "loss": 1.9866,
+      "step": 504
+    },
+    {
+      "epoch": 0.5534246575342465,
+      "grad_norm": 2.286147356033325,
+      "learning_rate": 4.559301380991065e-06,
+      "loss": 1.9805,
+      "step": 505
+    },
+    {
+      "epoch": 0.5545205479452054,
+      "grad_norm": 2.064378023147583,
+      "learning_rate": 4.5572705117790414e-06,
+      "loss": 1.9259,
+      "step": 506
+    },
+    {
+      "epoch": 0.5556164383561644,
+      "grad_norm": 2.4020493030548096,
+      "learning_rate": 4.5552396425670196e-06,
+      "loss": 1.9731,
+      "step": 507
+    },
+    {
+      "epoch": 0.5567123287671233,
+      "grad_norm": 2.11594557762146,
+      "learning_rate": 4.553208773354996e-06,
+      "loss": 2.0126,
+      "step": 508
+    },
+    {
+      "epoch": 0.5578082191780822,
+      "grad_norm": 2.4520673751831055,
+      "learning_rate": 4.551177904142974e-06,
+      "loss": 1.9396,
+      "step": 509
+    },
+    {
+      "epoch": 0.5589041095890411,
+      "grad_norm": 2.302771806716919,
+      "learning_rate": 4.5491470349309505e-06,
+      "loss": 2.0361,
+      "step": 510
+    },
+    {
+      "epoch": 0.5589041095890411,
+      "eval_loss": 2.1408755779266357,
+      "eval_runtime": 36.9881,
+      "eval_samples_per_second": 27.036,
+      "eval_steps_per_second": 1.136,
+      "step": 510
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 2.250927686691284,
+      "learning_rate": 4.547116165718929e-06,
+      "loss": 1.9616,
+      "step": 511
+    },
+    {
+      "epoch": 0.5610958904109589,
+      "grad_norm": 2.4257938861846924,
+      "learning_rate": 4.545085296506905e-06,
+      "loss": 1.9836,
+      "step": 512
+    },
+    {
+      "epoch": 0.5621917808219178,
+      "grad_norm": 2.08693265914917,
+      "learning_rate": 4.543054427294882e-06,
+      "loss": 1.9541,
+      "step": 513
+    },
+    {
+      "epoch": 0.5632876712328767,
+      "grad_norm": 2.133875846862793,
+      "learning_rate": 4.54102355808286e-06,
+      "loss": 1.9377,
+      "step": 514
+    },
+    {
+      "epoch": 0.5643835616438356,
+      "grad_norm": 2.1217594146728516,
+      "learning_rate": 4.538992688870837e-06,
+      "loss": 1.9369,
+      "step": 515
+    },
+    {
+      "epoch": 0.5654794520547946,
+      "grad_norm": 2.117994546890259,
+      "learning_rate": 4.536961819658814e-06,
+      "loss": 1.9666,
+      "step": 516
+    },
+    {
+      "epoch": 0.5665753424657535,
+      "grad_norm": 2.1514523029327393,
+      "learning_rate": 4.5349309504467915e-06,
+      "loss": 1.9704,
+      "step": 517
+    },
+    {
+      "epoch": 0.5676712328767123,
+      "grad_norm": 2.2309377193450928,
+      "learning_rate": 4.532900081234769e-06,
+      "loss": 1.9858,
+      "step": 518
+    },
+    {
+      "epoch": 0.5687671232876712,
+      "grad_norm": 2.100398540496826,
+      "learning_rate": 4.530869212022746e-06,
+      "loss": 1.9573,
+      "step": 519
+    },
+    {
+      "epoch": 0.5698630136986301,
+      "grad_norm": 2.122840166091919,
+      "learning_rate": 4.528838342810723e-06,
+      "loss": 1.9813,
+      "step": 520
+    },
+    {
+      "epoch": 0.5698630136986301,
+      "eval_loss": 2.135481357574463,
+      "eval_runtime": 36.893,
+      "eval_samples_per_second": 27.105,
+      "eval_steps_per_second": 1.138,
+      "step": 520
+    },
+    {
+      "epoch": 0.570958904109589,
+      "grad_norm": 2.1103999614715576,
+      "learning_rate": 4.526807473598701e-06,
+      "loss": 1.9691,
+      "step": 521
+    },
+    {
+      "epoch": 0.572054794520548,
+      "grad_norm": 2.059232473373413,
+      "learning_rate": 4.524776604386678e-06,
+      "loss": 1.9687,
+      "step": 522
+    },
+    {
+      "epoch": 0.5731506849315069,
+      "grad_norm": 2.207000732421875,
+      "learning_rate": 4.522745735174655e-06,
+      "loss": 1.949,
+      "step": 523
+    },
+    {
+      "epoch": 0.5742465753424657,
+      "grad_norm": 2.264000177383423,
+      "learning_rate": 4.520714865962632e-06,
+      "loss": 2.0066,
+      "step": 524
+    },
+    {
+      "epoch": 0.5753424657534246,
+      "grad_norm": 2.121701717376709,
+      "learning_rate": 4.51868399675061e-06,
+      "loss": 1.9879,
+      "step": 525
+    },
+    {
+      "epoch": 0.5764383561643835,
+      "grad_norm": 2.0745644569396973,
+      "learning_rate": 4.516653127538587e-06,
+      "loss": 1.9924,
+      "step": 526
+    },
+    {
+      "epoch": 0.5775342465753425,
+      "grad_norm": 2.2257349491119385,
+      "learning_rate": 4.514622258326564e-06,
+      "loss": 1.9525,
+      "step": 527
+    },
+    {
+      "epoch": 0.5786301369863014,
+      "grad_norm": 2.1678507328033447,
+      "learning_rate": 4.5125913891145415e-06,
+      "loss": 1.9378,
+      "step": 528
+    },
+    {
+      "epoch": 0.5797260273972603,
+      "grad_norm": 2.2800841331481934,
+      "learning_rate": 4.510560519902519e-06,
+      "loss": 1.9685,
+      "step": 529
+    },
+    {
+      "epoch": 0.5808219178082191,
+      "grad_norm": 2.1471869945526123,
+      "learning_rate": 4.508529650690496e-06,
+      "loss": 2.0027,
+      "step": 530
+    },
+    {
+      "epoch": 0.5808219178082191,
+      "eval_loss": 2.1354892253875732,
+      "eval_runtime": 36.8434,
+      "eval_samples_per_second": 27.142,
+      "eval_steps_per_second": 1.14,
+      "step": 530
+    },
+    {
+      "epoch": 0.581917808219178,
+      "grad_norm": 2.2238621711730957,
+      "learning_rate": 4.506498781478473e-06,
+      "loss": 1.9258,
+      "step": 531
+    },
+    {
+      "epoch": 0.583013698630137,
+      "grad_norm": 2.2224624156951904,
+      "learning_rate": 4.504467912266451e-06,
+      "loss": 1.9168,
+      "step": 532
+    },
+    {
+      "epoch": 0.5841095890410959,
+      "grad_norm": 1.9850294589996338,
+      "learning_rate": 4.502437043054428e-06,
+      "loss": 1.9656,
+      "step": 533
+    },
+    {
+      "epoch": 0.5852054794520548,
+      "grad_norm": 2.1303517818450928,
+      "learning_rate": 4.500406173842405e-06,
+      "loss": 2.0349,
+      "step": 534
+    },
+    {
+      "epoch": 0.5863013698630137,
+      "grad_norm": 2.2114832401275635,
+      "learning_rate": 4.4983753046303824e-06,
+      "loss": 1.9878,
+      "step": 535
+    },
+    {
+      "epoch": 0.5873972602739727,
+      "grad_norm": 1.9375886917114258,
+      "learning_rate": 4.49634443541836e-06,
+      "loss": 1.9724,
+      "step": 536
+    },
+    {
+      "epoch": 0.5884931506849315,
+      "grad_norm": 2.185518741607666,
+      "learning_rate": 4.494313566206336e-06,
+      "loss": 1.9584,
+      "step": 537
+    },
+    {
+      "epoch": 0.5895890410958904,
+      "grad_norm": 2.125638484954834,
+      "learning_rate": 4.492282696994314e-06,
+      "loss": 2.0203,
+      "step": 538
+    },
+    {
+      "epoch": 0.5906849315068493,
+      "grad_norm": 1.9456262588500977,
+      "learning_rate": 4.490251827782291e-06,
+      "loss": 1.9523,
+      "step": 539
+    },
+    {
+      "epoch": 0.5917808219178082,
+      "grad_norm": 2.088125705718994,
+      "learning_rate": 4.488220958570269e-06,
+      "loss": 1.9271,
+      "step": 540
+    },
+    {
+      "epoch": 0.5917808219178082,
+      "eval_loss": 2.134958505630493,
+      "eval_runtime": 36.8342,
+      "eval_samples_per_second": 27.149,
+      "eval_steps_per_second": 1.14,
+      "step": 540
+    },
+    {
+      "epoch": 0.5928767123287672,
+      "grad_norm": 2.1423559188842773,
+      "learning_rate": 4.486190089358245e-06,
+      "loss": 1.9467,
+      "step": 541
+    },
+    {
+      "epoch": 0.5939726027397261,
+      "grad_norm": 2.2169737815856934,
+      "learning_rate": 4.484159220146223e-06,
+      "loss": 1.8901,
+      "step": 542
+    },
+    {
+      "epoch": 0.5950684931506849,
+      "grad_norm": 2.419947385787964,
+      "learning_rate": 4.4821283509342e-06,
+      "loss": 2.0064,
+      "step": 543
+    },
+    {
+      "epoch": 0.5961643835616438,
+      "grad_norm": 2.0396435260772705,
+      "learning_rate": 4.480097481722178e-06,
+      "loss": 1.9819,
+      "step": 544
+    },
+    {
+      "epoch": 0.5972602739726027,
+      "grad_norm": 2.210700511932373,
+      "learning_rate": 4.478066612510154e-06,
+      "loss": 1.9578,
+      "step": 545
+    },
+    {
+      "epoch": 0.5983561643835617,
+      "grad_norm": 2.159698009490967,
+      "learning_rate": 4.4760357432981325e-06,
+      "loss": 1.9313,
+      "step": 546
+    },
+    {
+      "epoch": 0.5994520547945206,
+      "grad_norm": 2.125058650970459,
+      "learning_rate": 4.474004874086109e-06,
+      "loss": 1.9549,
+      "step": 547
+    },
+    {
+      "epoch": 0.6005479452054795,
+      "grad_norm": 2.477320432662964,
+      "learning_rate": 4.471974004874087e-06,
+      "loss": 2.0883,
+      "step": 548
+    },
+    {
+      "epoch": 0.6016438356164383,
+      "grad_norm": 2.214901924133301,
+      "learning_rate": 4.4699431356620635e-06,
+      "loss": 1.9184,
+      "step": 549
+    },
+    {
+      "epoch": 0.6027397260273972,
+      "grad_norm": 2.1001312732696533,
+      "learning_rate": 4.467912266450041e-06,
+      "loss": 2.0231,
+      "step": 550
+    },
+    {
+      "epoch": 0.6027397260273972,
+      "eval_loss": 2.1315157413482666,
+      "eval_runtime": 36.8131,
+      "eval_samples_per_second": 27.164,
+      "eval_steps_per_second": 1.141,
+      "step": 550
+    },
+    {
+      "epoch": 0.6038356164383561,
+      "grad_norm": 2.1030538082122803,
+      "learning_rate": 4.465881397238018e-06,
+      "loss": 1.967,
+      "step": 551
+    },
+    {
+      "epoch": 0.6049315068493151,
+      "grad_norm": 2.0390465259552,
+      "learning_rate": 4.463850528025995e-06,
+      "loss": 1.9615,
+      "step": 552
+    },
+    {
+      "epoch": 0.606027397260274,
+      "grad_norm": 2.1331627368927,
+      "learning_rate": 4.4618196588139726e-06,
+      "loss": 1.9667,
+      "step": 553
+    },
+    {
+      "epoch": 0.6071232876712329,
+      "grad_norm": 2.1987671852111816,
+      "learning_rate": 4.45978878960195e-06,
+      "loss": 1.9922,
+      "step": 554
+    },
+    {
+      "epoch": 0.6082191780821918,
+      "grad_norm": 2.127434253692627,
+      "learning_rate": 4.457757920389927e-06,
+      "loss": 1.9913,
+      "step": 555
+    },
+    {
+      "epoch": 0.6093150684931506,
+      "grad_norm": 2.2486324310302734,
+      "learning_rate": 4.455727051177904e-06,
+      "loss": 1.9239,
+      "step": 556
+    },
+    {
+      "epoch": 0.6104109589041096,
+      "grad_norm": 2.127575397491455,
+      "learning_rate": 4.453696181965882e-06,
+      "loss": 1.9521,
+      "step": 557
+    },
+    {
+      "epoch": 0.6115068493150685,
+      "grad_norm": 2.0798254013061523,
+      "learning_rate": 4.451665312753859e-06,
+      "loss": 2.0085,
+      "step": 558
+    },
+    {
+      "epoch": 0.6126027397260274,
+      "grad_norm": 2.287574291229248,
+      "learning_rate": 4.449634443541836e-06,
+      "loss": 1.98,
+      "step": 559
+    },
+    {
+      "epoch": 0.6136986301369863,
+      "grad_norm": 2.2056541442871094,
+      "learning_rate": 4.4476035743298135e-06,
+      "loss": 2.0282,
+      "step": 560
+    },
+    {
+      "epoch": 0.6136986301369863,
+      "eval_loss": 2.1287026405334473,
+      "eval_runtime": 36.7788,
+      "eval_samples_per_second": 27.19,
+      "eval_steps_per_second": 1.142,
+      "step": 560
+    },
+    {
+      "epoch": 0.6147945205479453,
+      "grad_norm": 2.169157028198242,
+      "learning_rate": 4.445572705117791e-06,
+      "loss": 1.9308,
+      "step": 561
+    },
+    {
+      "epoch": 0.6158904109589041,
+      "grad_norm": 2.1035149097442627,
+      "learning_rate": 4.443541835905768e-06,
+      "loss": 1.9507,
+      "step": 562
+    },
+    {
+      "epoch": 0.616986301369863,
+      "grad_norm": 2.1174328327178955,
+      "learning_rate": 4.441510966693745e-06,
+      "loss": 2.0162,
+      "step": 563
+    },
+    {
+      "epoch": 0.6180821917808219,
+      "grad_norm": 2.201641321182251,
+      "learning_rate": 4.439480097481723e-06,
+      "loss": 1.9573,
+      "step": 564
+    },
+    {
+      "epoch": 0.6191780821917808,
+      "grad_norm": 2.1763603687286377,
+      "learning_rate": 4.4374492282697e-06,
+      "loss": 2.0149,
+      "step": 565
+    },
+    {
+      "epoch": 0.6202739726027398,
+      "grad_norm": 2.2866439819335938,
+      "learning_rate": 4.435418359057677e-06,
+      "loss": 1.8836,
+      "step": 566
+    },
+    {
+      "epoch": 0.6213698630136987,
+      "grad_norm": 2.2060160636901855,
+      "learning_rate": 4.433387489845654e-06,
+      "loss": 1.9806,
+      "step": 567
+    },
+    {
+      "epoch": 0.6224657534246575,
+      "grad_norm": 2.0614662170410156,
+      "learning_rate": 4.431356620633632e-06,
+      "loss": 1.9821,
+      "step": 568
+    },
+    {
+      "epoch": 0.6235616438356164,
+      "grad_norm": 2.1055402755737305,
+      "learning_rate": 4.429325751421609e-06,
+      "loss": 1.9365,
+      "step": 569
+    },
+    {
+      "epoch": 0.6246575342465753,
+      "grad_norm": 1.9907679557800293,
+      "learning_rate": 4.427294882209586e-06,
+      "loss": 1.9429,
+      "step": 570
+    },
+    {
+      "epoch": 0.6246575342465753,
+      "eval_loss": 2.1286206245422363,
+      "eval_runtime": 36.7748,
+      "eval_samples_per_second": 27.193,
+      "eval_steps_per_second": 1.142,
+      "step": 570
+    },
+    {
+      "epoch": 0.6257534246575343,
+      "grad_norm": 2.0169243812561035,
+      "learning_rate": 4.4252640129975635e-06,
+      "loss": 1.9643,
+      "step": 571
+    },
+    {
+      "epoch": 0.6268493150684932,
+      "grad_norm": 2.1082401275634766,
+      "learning_rate": 4.423233143785541e-06,
+      "loss": 1.9009,
+      "step": 572
+    },
+    {
+      "epoch": 0.6279452054794521,
+      "grad_norm": 2.202542543411255,
+      "learning_rate": 4.421202274573518e-06,
+      "loss": 1.9694,
+      "step": 573
+    },
+    {
+      "epoch": 0.6290410958904109,
+      "grad_norm": 2.028489828109741,
+      "learning_rate": 4.4191714053614945e-06,
+      "loss": 1.9697,
+      "step": 574
+    },
+    {
+      "epoch": 0.6301369863013698,
+      "grad_norm": 2.134274482727051,
+      "learning_rate": 4.417140536149473e-06,
+      "loss": 1.8672,
+      "step": 575
+    },
+    {
+      "epoch": 0.6312328767123287,
+      "grad_norm": 2.063542366027832,
+      "learning_rate": 4.415109666937449e-06,
+      "loss": 1.909,
+      "step": 576
+    },
+    {
+      "epoch": 0.6323287671232877,
+      "grad_norm": 1.9747039079666138,
+      "learning_rate": 4.413078797725427e-06,
+      "loss": 1.9795,
+      "step": 577
+    },
+    {
+      "epoch": 0.6334246575342466,
+      "grad_norm": 2.063673496246338,
+      "learning_rate": 4.411047928513404e-06,
+      "loss": 1.9283,
+      "step": 578
+    },
+    {
+      "epoch": 0.6345205479452055,
+      "grad_norm": 2.024611711502075,
+      "learning_rate": 4.409017059301382e-06,
+      "loss": 1.9682,
+      "step": 579
+    },
+    {
+      "epoch": 0.6356164383561644,
+      "grad_norm": 2.0471599102020264,
+      "learning_rate": 4.406986190089358e-06,
+      "loss": 1.969,
+      "step": 580
+    },
+    {
+      "epoch": 0.6356164383561644,
+      "eval_loss": 2.1233692169189453,
+      "eval_runtime": 36.8185,
+      "eval_samples_per_second": 27.16,
+      "eval_steps_per_second": 1.141,
+      "step": 580
+    },
+    {
+      "epoch": 0.6367123287671232,
+      "grad_norm": 2.1038553714752197,
+      "learning_rate": 4.404955320877336e-06,
+      "loss": 1.9915,
+      "step": 581
+    },
+    {
+      "epoch": 0.6378082191780822,
+      "grad_norm": 2.1222152709960938,
+      "learning_rate": 4.402924451665313e-06,
+      "loss": 1.9892,
+      "step": 582
+    },
+    {
+      "epoch": 0.6389041095890411,
+      "grad_norm": 2.09137225151062,
+      "learning_rate": 4.400893582453291e-06,
+      "loss": 1.934,
+      "step": 583
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 2.2158281803131104,
+      "learning_rate": 4.398862713241267e-06,
+      "loss": 1.9188,
+      "step": 584
+    },
+    {
+      "epoch": 0.6410958904109589,
+      "grad_norm": 2.34619140625,
+      "learning_rate": 4.396831844029245e-06,
+      "loss": 1.9495,
+      "step": 585
+    },
+    {
+      "epoch": 0.6421917808219179,
+      "grad_norm": 2.2732315063476562,
+      "learning_rate": 4.394800974817222e-06,
+      "loss": 1.9632,
+      "step": 586
+    },
+    {
+      "epoch": 0.6432876712328767,
+      "grad_norm": 2.153338670730591,
+      "learning_rate": 4.392770105605199e-06,
+      "loss": 1.9551,
+      "step": 587
+    },
+    {
+      "epoch": 0.6443835616438356,
+      "grad_norm": 2.2201521396636963,
+      "learning_rate": 4.390739236393176e-06,
+      "loss": 1.9677,
+      "step": 588
+    },
+    {
+      "epoch": 0.6454794520547945,
+      "grad_norm": 1.9894570112228394,
+      "learning_rate": 4.388708367181154e-06,
+      "loss": 1.9826,
+      "step": 589
+    },
+    {
+      "epoch": 0.6465753424657534,
+      "grad_norm": 2.145061492919922,
+      "learning_rate": 4.386677497969131e-06,
+      "loss": 1.9826,
+      "step": 590
+    },
+    {
+      "epoch": 0.6465753424657534,
+      "eval_loss": 2.122312545776367,
+      "eval_runtime": 37.1153,
+      "eval_samples_per_second": 26.943,
+      "eval_steps_per_second": 1.132,
+      "step": 590
+    },
+    {
+      "epoch": 0.6476712328767124,
+      "grad_norm": 2.257237672805786,
+      "learning_rate": 4.384646628757108e-06,
+      "loss": 1.9722,
+      "step": 591
+    },
+    {
+      "epoch": 0.6487671232876713,
+      "grad_norm": 2.2406375408172607,
+      "learning_rate": 4.3826157595450855e-06,
+      "loss": 2.0141,
+      "step": 592
+    },
+    {
+      "epoch": 0.6498630136986301,
+      "grad_norm": 1.9656853675842285,
+      "learning_rate": 4.380584890333063e-06,
+      "loss": 1.9435,
+      "step": 593
+    },
+    {
+      "epoch": 0.650958904109589,
+      "grad_norm": 2.1063120365142822,
+      "learning_rate": 4.37855402112104e-06,
+      "loss": 1.9001,
+      "step": 594
+    },
+    {
+      "epoch": 0.6520547945205479,
+      "grad_norm": 2.1594269275665283,
+      "learning_rate": 4.376523151909017e-06,
+      "loss": 1.8596,
+      "step": 595
+    },
+    {
+      "epoch": 0.6531506849315069,
+      "grad_norm": 2.2261908054351807,
+      "learning_rate": 4.3744922826969946e-06,
+      "loss": 2.016,
+      "step": 596
+    },
+    {
+      "epoch": 0.6542465753424658,
+      "grad_norm": 1.9940093755722046,
+      "learning_rate": 4.372461413484972e-06,
+      "loss": 1.9293,
+      "step": 597
+    },
+    {
+      "epoch": 0.6553424657534247,
+      "grad_norm": 2.62186861038208,
+      "learning_rate": 4.370430544272949e-06,
+      "loss": 1.9656,
+      "step": 598
+    },
+    {
+      "epoch": 0.6564383561643835,
+      "grad_norm": 2.3531494140625,
+      "learning_rate": 4.368399675060926e-06,
+      "loss": 1.9984,
+      "step": 599
+    },
+    {
+      "epoch": 0.6575342465753424,
+      "grad_norm": 2.362769365310669,
+      "learning_rate": 4.366368805848904e-06,
+      "loss": 1.9971,
+      "step": 600
+    },
+    {
+      "epoch": 0.6575342465753424,
+      "eval_loss": 2.1172940731048584,
+      "eval_runtime": 37.0498,
+      "eval_samples_per_second": 26.991,
+      "eval_steps_per_second": 1.134,
+      "step": 600
+    },
+    {
+      "epoch": 0.6586301369863014,
+      "grad_norm": 2.3806607723236084,
+      "learning_rate": 4.364337936636881e-06,
+      "loss": 1.9566,
+      "step": 601
+    },
+    {
+      "epoch": 0.6597260273972603,
+      "grad_norm": 2.5531814098358154,
+      "learning_rate": 4.362307067424858e-06,
+      "loss": 1.9002,
+      "step": 602
+    },
+    {
+      "epoch": 0.6608219178082192,
+      "grad_norm": 2.283364772796631,
+      "learning_rate": 4.3602761982128355e-06,
+      "loss": 1.9633,
+      "step": 603
+    },
+    {
+      "epoch": 0.6619178082191781,
+      "grad_norm": 2.180171489715576,
+      "learning_rate": 4.358245329000813e-06,
+      "loss": 1.9277,
+      "step": 604
+    },
+    {
+      "epoch": 0.663013698630137,
+      "grad_norm": 2.0771350860595703,
+      "learning_rate": 4.35621445978879e-06,
+      "loss": 1.9484,
+      "step": 605
+    },
+    {
+      "epoch": 0.6641095890410958,
+      "grad_norm": 2.3772811889648438,
+      "learning_rate": 4.354183590576767e-06,
+      "loss": 1.9564,
+      "step": 606
+    },
+    {
+      "epoch": 0.6652054794520548,
+      "grad_norm": 2.205486297607422,
+      "learning_rate": 4.352152721364745e-06,
+      "loss": 1.9389,
+      "step": 607
+    },
+    {
+      "epoch": 0.6663013698630137,
+      "grad_norm": 2.085190773010254,
+      "learning_rate": 4.350121852152722e-06,
+      "loss": 1.8948,
+      "step": 608
+    },
+    {
+      "epoch": 0.6673972602739726,
+      "grad_norm": 2.4398982524871826,
+      "learning_rate": 4.348090982940699e-06,
+      "loss": 1.9736,
+      "step": 609
+    },
+    {
+      "epoch": 0.6684931506849315,
+      "grad_norm": 2.194087028503418,
+      "learning_rate": 4.3460601137286764e-06,
+      "loss": 1.9607,
+      "step": 610
+    },
+    {
+      "epoch": 0.6684931506849315,
+      "eval_loss": 2.115041971206665,
+      "eval_runtime": 36.9738,
+      "eval_samples_per_second": 27.046,
+      "eval_steps_per_second": 1.136,
+      "step": 610
+    },
+    {
+      "epoch": 0.6695890410958905,
+      "grad_norm": 2.143054485321045,
+      "learning_rate": 4.344029244516654e-06,
+      "loss": 1.9733,
+      "step": 611
+    },
+    {
+      "epoch": 0.6706849315068493,
+      "grad_norm": 2.2484617233276367,
+      "learning_rate": 4.341998375304631e-06,
+      "loss": 1.9417,
+      "step": 612
+    },
+    {
+      "epoch": 0.6717808219178082,
+      "grad_norm": 2.0430805683135986,
+      "learning_rate": 4.339967506092607e-06,
+      "loss": 2.0192,
+      "step": 613
+    },
+    {
+      "epoch": 0.6728767123287671,
+      "grad_norm": 2.061089277267456,
+      "learning_rate": 4.3379366368805855e-06,
+      "loss": 1.9543,
+      "step": 614
+    },
+    {
+      "epoch": 0.673972602739726,
+      "grad_norm": 2.298022747039795,
+      "learning_rate": 4.335905767668562e-06,
+      "loss": 1.9579,
+      "step": 615
+    },
+    {
+      "epoch": 0.675068493150685,
+      "grad_norm": 2.1418962478637695,
+      "learning_rate": 4.33387489845654e-06,
+      "loss": 1.938,
+      "step": 616
+    },
+    {
+      "epoch": 0.6761643835616439,
+      "grad_norm": 2.1005501747131348,
+      "learning_rate": 4.3318440292445165e-06,
+      "loss": 1.972,
+      "step": 617
+    },
+    {
+      "epoch": 0.6772602739726027,
+      "grad_norm": 2.1665523052215576,
+      "learning_rate": 4.329813160032495e-06,
+      "loss": 1.9535,
+      "step": 618
+    },
+    {
+      "epoch": 0.6783561643835616,
+      "grad_norm": 2.144580841064453,
+      "learning_rate": 4.327782290820471e-06,
+      "loss": 1.9188,
+      "step": 619
+    },
+    {
+      "epoch": 0.6794520547945205,
+      "grad_norm": 1.9855908155441284,
+      "learning_rate": 4.325751421608449e-06,
+      "loss": 1.912,
+      "step": 620
+    },
+    {
+      "epoch": 0.6794520547945205,
+      "eval_loss": 2.118319272994995,
+      "eval_runtime": 36.9677,
+      "eval_samples_per_second": 27.051,
+      "eval_steps_per_second": 1.136,
+      "step": 620
+    },
+    {
+      "epoch": 0.6805479452054795,
+      "grad_norm": 2.395538091659546,
+      "learning_rate": 4.323720552396426e-06,
+      "loss": 1.9886,
+      "step": 621
+    },
+    {
+      "epoch": 0.6816438356164384,
+      "grad_norm": 2.190574884414673,
+      "learning_rate": 4.321689683184404e-06,
+      "loss": 1.9326,
+      "step": 622
+    },
+    {
+      "epoch": 0.6827397260273973,
+      "grad_norm": 2.1826560497283936,
+      "learning_rate": 4.31965881397238e-06,
+      "loss": 1.9278,
+      "step": 623
+    },
+    {
+      "epoch": 0.6838356164383562,
+      "grad_norm": 2.151097536087036,
+      "learning_rate": 4.3176279447603574e-06,
+      "loss": 1.9359,
+      "step": 624
+    },
+    {
+      "epoch": 0.684931506849315,
+      "grad_norm": 2.1830122470855713,
+      "learning_rate": 4.315597075548335e-06,
+      "loss": 1.9728,
+      "step": 625
+    },
+    {
+      "epoch": 0.686027397260274,
+      "grad_norm": 2.203325033187866,
+      "learning_rate": 4.313566206336312e-06,
+      "loss": 1.9638,
+      "step": 626
+    },
+    {
+      "epoch": 0.6871232876712329,
+      "grad_norm": 2.7240309715270996,
+      "learning_rate": 4.311535337124289e-06,
+      "loss": 1.9907,
+      "step": 627
+    },
+    {
+      "epoch": 0.6882191780821918,
+      "grad_norm": 2.133599281311035,
+      "learning_rate": 4.3095044679122665e-06,
+      "loss": 1.9352,
+      "step": 628
+    },
+    {
+      "epoch": 0.6893150684931507,
+      "grad_norm": 2.0318551063537598,
+      "learning_rate": 4.307473598700244e-06,
+      "loss": 1.9432,
+      "step": 629
+    },
+    {
+      "epoch": 0.6904109589041096,
+      "grad_norm": 2.502101182937622,
+      "learning_rate": 4.305442729488221e-06,
+      "loss": 1.9641,
+      "step": 630
+    },
+    {
+      "epoch": 0.6904109589041096,
+      "eval_loss": 2.1158926486968994,
+      "eval_runtime": 36.8818,
+      "eval_samples_per_second": 27.114,
+      "eval_steps_per_second": 1.139,
+      "step": 630
+    },
+    {
+      "epoch": 0.6915068493150685,
+      "grad_norm": 2.3620944023132324,
+      "learning_rate": 4.303411860276198e-06,
+      "loss": 1.8841,
+      "step": 631
+    },
+    {
+      "epoch": 0.6926027397260274,
+      "grad_norm": 2.3139240741729736,
+      "learning_rate": 4.301380991064176e-06,
+      "loss": 1.9896,
+      "step": 632
+    },
+    {
+      "epoch": 0.6936986301369863,
+      "grad_norm": 2.314877510070801,
+      "learning_rate": 4.299350121852153e-06,
+      "loss": 1.9655,
+      "step": 633
+    },
+    {
+      "epoch": 0.6947945205479452,
+      "grad_norm": 2.036371946334839,
+      "learning_rate": 4.29731925264013e-06,
+      "loss": 1.9917,
+      "step": 634
+    },
+    {
+      "epoch": 0.6958904109589041,
+      "grad_norm": 2.1859662532806396,
+      "learning_rate": 4.2952883834281075e-06,
+      "loss": 1.9805,
+      "step": 635
+    },
+    {
+      "epoch": 0.6969863013698631,
+      "grad_norm": 2.2317097187042236,
+      "learning_rate": 4.293257514216085e-06,
+      "loss": 1.9402,
+      "step": 636
+    },
+    {
+      "epoch": 0.6980821917808219,
+      "grad_norm": 2.3249905109405518,
+      "learning_rate": 4.291226645004062e-06,
+      "loss": 1.9013,
+      "step": 637
+    },
+    {
+      "epoch": 0.6991780821917808,
+      "grad_norm": 2.199758768081665,
+      "learning_rate": 4.289195775792039e-06,
+      "loss": 1.9783,
+      "step": 638
+    },
+    {
+      "epoch": 0.7002739726027397,
+      "grad_norm": 2.3598549365997314,
+      "learning_rate": 4.287164906580017e-06,
+      "loss": 1.9936,
+      "step": 639
+    },
+    {
+      "epoch": 0.7013698630136986,
+      "grad_norm": 2.1705522537231445,
+      "learning_rate": 4.285134037367994e-06,
+      "loss": 1.9304,
+      "step": 640
+    },
+    {
+      "epoch": 0.7013698630136986,
+      "eval_loss": 2.1055190563201904,
+      "eval_runtime": 36.8673,
+      "eval_samples_per_second": 27.124,
+      "eval_steps_per_second": 1.139,
+      "step": 640
+    },
+    {
+      "epoch": 0.7024657534246576,
+      "grad_norm": 2.3253612518310547,
+      "learning_rate": 4.283103168155971e-06,
+      "loss": 1.9196,
+      "step": 641
+    },
+    {
+      "epoch": 0.7035616438356165,
+      "grad_norm": 2.3054087162017822,
+      "learning_rate": 4.281072298943948e-06,
+      "loss": 1.9053,
+      "step": 642
+    },
+    {
+      "epoch": 0.7046575342465753,
+      "grad_norm": 2.262057304382324,
+      "learning_rate": 4.279041429731926e-06,
+      "loss": 1.8999,
+      "step": 643
+    },
+    {
+      "epoch": 0.7057534246575342,
+      "grad_norm": 2.421391248703003,
+      "learning_rate": 4.277010560519903e-06,
+      "loss": 1.9438,
+      "step": 644
+    },
+    {
+      "epoch": 0.7068493150684931,
+      "grad_norm": 2.2841498851776123,
+      "learning_rate": 4.27497969130788e-06,
+      "loss": 1.9187,
+      "step": 645
+    },
+    {
+      "epoch": 0.707945205479452,
+      "grad_norm": 2.3226938247680664,
+      "learning_rate": 4.2729488220958575e-06,
+      "loss": 1.9275,
+      "step": 646
+    },
+    {
+      "epoch": 0.709041095890411,
+      "grad_norm": 2.2306301593780518,
+      "learning_rate": 4.270917952883835e-06,
+      "loss": 1.9684,
+      "step": 647
+    },
+    {
+      "epoch": 0.7101369863013699,
+      "grad_norm": 2.0551652908325195,
+      "learning_rate": 4.268887083671812e-06,
+      "loss": 1.9849,
+      "step": 648
+    },
+    {
+      "epoch": 0.7112328767123288,
+      "grad_norm": 2.2952210903167725,
+      "learning_rate": 4.266856214459789e-06,
+      "loss": 1.9026,
+      "step": 649
+    },
+    {
+      "epoch": 0.7123287671232876,
+      "grad_norm": 2.1630797386169434,
+      "learning_rate": 4.264825345247766e-06,
+      "loss": 2.0184,
+      "step": 650
+    },
+    {
+      "epoch": 0.7123287671232876,
+      "eval_loss": 2.1037871837615967,
+      "eval_runtime": 36.8221,
+      "eval_samples_per_second": 27.158,
+      "eval_steps_per_second": 1.141,
+      "step": 650
+    },
+    {
+      "epoch": 0.7134246575342466,
+      "grad_norm": 1.964150309562683,
+      "learning_rate": 4.262794476035744e-06,
+      "loss": 1.8932,
+      "step": 651
+    },
+    {
+      "epoch": 0.7145205479452055,
+      "grad_norm": 2.1474685668945312,
+      "learning_rate": 4.26076360682372e-06,
+      "loss": 1.9383,
+      "step": 652
+    },
+    {
+      "epoch": 0.7156164383561644,
+      "grad_norm": 2.199824094772339,
+      "learning_rate": 4.2587327376116984e-06,
+      "loss": 1.8599,
+      "step": 653
+    },
+    {
+      "epoch": 0.7167123287671233,
+      "grad_norm": 2.12156343460083,
+      "learning_rate": 4.256701868399675e-06,
+      "loss": 1.9162,
+      "step": 654
+    },
+    {
+      "epoch": 0.7178082191780822,
+      "grad_norm": 2.3832154273986816,
+      "learning_rate": 4.254670999187653e-06,
+      "loss": 1.9613,
+      "step": 655
+    },
+    {
+      "epoch": 0.718904109589041,
+      "grad_norm": 2.063906192779541,
+      "learning_rate": 4.252640129975629e-06,
+      "loss": 1.9421,
+      "step": 656
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 2.184842586517334,
+      "learning_rate": 4.2506092607636075e-06,
+      "loss": 1.9471,
+      "step": 657
+    },
+    {
+      "epoch": 0.7210958904109589,
+      "grad_norm": 2.1340036392211914,
+      "learning_rate": 4.248578391551584e-06,
+      "loss": 1.931,
+      "step": 658
+    },
+    {
+      "epoch": 0.7221917808219178,
+      "grad_norm": 2.117544651031494,
+      "learning_rate": 4.246547522339562e-06,
+      "loss": 1.8698,
+      "step": 659
+    },
+    {
+      "epoch": 0.7232876712328767,
+      "grad_norm": 2.308344841003418,
+      "learning_rate": 4.2445166531275385e-06,
+      "loss": 1.9217,
+      "step": 660
+    },
+    {
+      "epoch": 0.7232876712328767,
+      "eval_loss": 2.1086323261260986,
+      "eval_runtime": 36.8082,
+      "eval_samples_per_second": 27.168,
+      "eval_steps_per_second": 1.141,
+      "step": 660
+    },
+    {
+      "epoch": 0.7243835616438357,
+      "grad_norm": 2.2239017486572266,
+      "learning_rate": 4.242485783915517e-06,
+      "loss": 1.9286,
+      "step": 661
+    },
+    {
+      "epoch": 0.7254794520547945,
+      "grad_norm": 2.263697624206543,
+      "learning_rate": 4.240454914703493e-06,
+      "loss": 1.8821,
+      "step": 662
+    },
+    {
+      "epoch": 0.7265753424657534,
+      "grad_norm": 2.4305059909820557,
+      "learning_rate": 4.23842404549147e-06,
+      "loss": 1.8996,
+      "step": 663
+    },
+    {
+      "epoch": 0.7276712328767123,
+      "grad_norm": 2.2333695888519287,
+      "learning_rate": 4.236393176279448e-06,
+      "loss": 1.8906,
+      "step": 664
+    },
+    {
+      "epoch": 0.7287671232876712,
+      "grad_norm": 2.160982608795166,
+      "learning_rate": 4.234362307067425e-06,
+      "loss": 1.879,
+      "step": 665
+    },
+    {
+      "epoch": 0.7298630136986302,
+      "grad_norm": 2.3707094192504883,
+      "learning_rate": 4.232331437855402e-06,
+      "loss": 1.9577,
+      "step": 666
+    },
+    {
+      "epoch": 0.7309589041095891,
+      "grad_norm": 2.0502827167510986,
+      "learning_rate": 4.2303005686433795e-06,
+      "loss": 1.9801,
+      "step": 667
+    },
+    {
+      "epoch": 0.7320547945205479,
+      "grad_norm": 2.0648176670074463,
+      "learning_rate": 4.228269699431357e-06,
+      "loss": 1.978,
+      "step": 668
+    },
+    {
+      "epoch": 0.7331506849315068,
+      "grad_norm": 2.346316337585449,
+      "learning_rate": 4.226238830219334e-06,
+      "loss": 1.9387,
+      "step": 669
+    },
+    {
+      "epoch": 0.7342465753424657,
+      "grad_norm": 1.9734236001968384,
+      "learning_rate": 4.224207961007311e-06,
+      "loss": 1.9396,
+      "step": 670
+    },
+    {
+      "epoch": 0.7342465753424657,
+      "eval_loss": 2.1104705333709717,
+      "eval_runtime": 36.8144,
+      "eval_samples_per_second": 27.163,
+      "eval_steps_per_second": 1.141,
+      "step": 670
+    },
+    {
+      "epoch": 0.7353424657534247,
+      "grad_norm": 2.3170549869537354,
+      "learning_rate": 4.2221770917952886e-06,
+      "loss": 1.9398,
+      "step": 671
+    },
+    {
+      "epoch": 0.7364383561643836,
+      "grad_norm": 2.0437264442443848,
+      "learning_rate": 4.220146222583266e-06,
+      "loss": 1.9284,
+      "step": 672
+    },
+    {
+      "epoch": 0.7375342465753425,
+      "grad_norm": 2.0808510780334473,
+      "learning_rate": 4.218115353371243e-06,
+      "loss": 1.9425,
+      "step": 673
+    },
+    {
+      "epoch": 0.7386301369863014,
+      "grad_norm": 2.0854742527008057,
+      "learning_rate": 4.21608448415922e-06,
+      "loss": 1.9358,
+      "step": 674
+    },
+    {
+      "epoch": 0.7397260273972602,
+      "grad_norm": 2.131622552871704,
+      "learning_rate": 4.214053614947198e-06,
+      "loss": 1.9048,
+      "step": 675
+    },
+    {
+      "epoch": 0.7408219178082192,
+      "grad_norm": 2.276359796524048,
+      "learning_rate": 4.212022745735175e-06,
+      "loss": 1.9871,
+      "step": 676
+    },
+    {
+      "epoch": 0.7419178082191781,
+      "grad_norm": 2.2194812297821045,
+      "learning_rate": 4.209991876523152e-06,
+      "loss": 1.882,
+      "step": 677
+    },
+    {
+      "epoch": 0.743013698630137,
+      "grad_norm": 2.181891679763794,
+      "learning_rate": 4.2079610073111295e-06,
+      "loss": 1.9546,
+      "step": 678
+    },
+    {
+      "epoch": 0.7441095890410959,
+      "grad_norm": 2.0956673622131348,
+      "learning_rate": 4.205930138099107e-06,
+      "loss": 1.8836,
+      "step": 679
+    },
+    {
+      "epoch": 0.7452054794520548,
+      "grad_norm": 2.057894229888916,
+      "learning_rate": 4.203899268887084e-06,
+      "loss": 1.9842,
+      "step": 680
+    },
+    {
+      "epoch": 0.7452054794520548,
+      "eval_loss": 2.104182720184326,
+      "eval_runtime": 36.8189,
+      "eval_samples_per_second": 27.16,
+      "eval_steps_per_second": 1.141,
+      "step": 680
+    },
+    {
+      "epoch": 0.7463013698630137,
+      "grad_norm": 2.0965662002563477,
+      "learning_rate": 4.201868399675061e-06,
+      "loss": 1.9378,
+      "step": 681
+    },
+    {
+      "epoch": 0.7473972602739726,
+      "grad_norm": 2.1843457221984863,
+      "learning_rate": 4.199837530463039e-06,
+      "loss": 1.9784,
+      "step": 682
+    },
+    {
+      "epoch": 0.7484931506849315,
+      "grad_norm": 2.2044246196746826,
+      "learning_rate": 4.197806661251016e-06,
+      "loss": 1.8642,
+      "step": 683
+    },
+    {
+      "epoch": 0.7495890410958904,
+      "grad_norm": 2.202115058898926,
+      "learning_rate": 4.195775792038993e-06,
+      "loss": 1.929,
+      "step": 684
+    },
+    {
+      "epoch": 0.7506849315068493,
+      "grad_norm": 2.3671462535858154,
+      "learning_rate": 4.19374492282697e-06,
+      "loss": 1.9172,
+      "step": 685
+    },
+    {
+      "epoch": 0.7517808219178083,
+      "grad_norm": 2.0602803230285645,
+      "learning_rate": 4.191714053614948e-06,
+      "loss": 1.8503,
+      "step": 686
+    },
+    {
+      "epoch": 0.7528767123287671,
+      "grad_norm": 2.12145733833313,
+      "learning_rate": 4.189683184402924e-06,
+      "loss": 1.9511,
+      "step": 687
+    },
+    {
+      "epoch": 0.753972602739726,
+      "grad_norm": 2.1049914360046387,
+      "learning_rate": 4.187652315190902e-06,
+      "loss": 1.9043,
+      "step": 688
+    },
+    {
+      "epoch": 0.7550684931506849,
+      "grad_norm": 2.235497236251831,
+      "learning_rate": 4.185621445978879e-06,
+      "loss": 1.9784,
+      "step": 689
+    },
+    {
+      "epoch": 0.7561643835616438,
+      "grad_norm": 2.173222303390503,
+      "learning_rate": 4.183590576766857e-06,
+      "loss": 1.9052,
+      "step": 690
+    },
+    {
+      "epoch": 0.7561643835616438,
+      "eval_loss": 2.0982251167297363,
+      "eval_runtime": 37.1446,
+      "eval_samples_per_second": 26.922,
+      "eval_steps_per_second": 1.131,
+      "step": 690
+    },
+    {
+      "epoch": 0.7572602739726028,
+      "grad_norm": 2.0693562030792236,
+      "learning_rate": 4.181559707554833e-06,
+      "loss": 1.9223,
+      "step": 691
+    },
+    {
+      "epoch": 0.7583561643835617,
+      "grad_norm": 2.209419012069702,
+      "learning_rate": 4.179528838342811e-06,
+      "loss": 1.9709,
+      "step": 692
+    },
+    {
+      "epoch": 0.7594520547945205,
+      "grad_norm": 2.066041946411133,
+      "learning_rate": 4.177497969130788e-06,
+      "loss": 1.889,
+      "step": 693
+    },
+    {
+      "epoch": 0.7605479452054794,
+      "grad_norm": 2.4114325046539307,
+      "learning_rate": 4.175467099918766e-06,
+      "loss": 1.9059,
+      "step": 694
+    },
+    {
+      "epoch": 0.7616438356164383,
+      "grad_norm": 2.0185043811798096,
+      "learning_rate": 4.173436230706742e-06,
+      "loss": 1.9877,
+      "step": 695
+    },
+    {
+      "epoch": 0.7627397260273973,
+      "grad_norm": 2.23392915725708,
+      "learning_rate": 4.1714053614947205e-06,
+      "loss": 1.9199,
+      "step": 696
+    },
+    {
+      "epoch": 0.7638356164383562,
+      "grad_norm": 2.2158737182617188,
+      "learning_rate": 4.169374492282697e-06,
+      "loss": 1.9001,
+      "step": 697
+    },
+    {
+      "epoch": 0.7649315068493151,
+      "grad_norm": 2.1511690616607666,
+      "learning_rate": 4.167343623070675e-06,
+      "loss": 1.9166,
+      "step": 698
+    },
+    {
+      "epoch": 0.766027397260274,
+      "grad_norm": 2.374741554260254,
+      "learning_rate": 4.165312753858652e-06,
+      "loss": 1.9684,
+      "step": 699
+    },
+    {
+      "epoch": 0.7671232876712328,
+      "grad_norm": 2.0409998893737793,
+      "learning_rate": 4.163281884646629e-06,
+      "loss": 1.9177,
+      "step": 700
+    },
+    {
+      "epoch": 0.7671232876712328,
+      "eval_loss": 2.096895933151245,
+      "eval_runtime": 37.066,
+      "eval_samples_per_second": 26.979,
+      "eval_steps_per_second": 1.133,
+      "step": 700
+    },
+    {
+      "epoch": 0.7682191780821918,
+      "grad_norm": 2.3173422813415527,
+      "learning_rate": 4.161251015434607e-06,
+      "loss": 1.9071,
+      "step": 701
+    },
+    {
+      "epoch": 0.7693150684931507,
+      "grad_norm": 2.077842950820923,
+      "learning_rate": 4.159220146222583e-06,
+      "loss": 1.935,
+      "step": 702
+    },
+    {
+      "epoch": 0.7704109589041096,
+      "grad_norm": 2.0518617630004883,
+      "learning_rate": 4.157189277010561e-06,
+      "loss": 1.8942,
+      "step": 703
+    },
+    {
+      "epoch": 0.7715068493150685,
+      "grad_norm": 1.9950882196426392,
+      "learning_rate": 4.155158407798538e-06,
+      "loss": 1.917,
+      "step": 704
+    },
+    {
+      "epoch": 0.7726027397260274,
+      "grad_norm": 2.3222219944000244,
+      "learning_rate": 4.153127538586516e-06,
+      "loss": 1.9551,
+      "step": 705
+    },
+    {
+      "epoch": 0.7736986301369863,
+      "grad_norm": 2.129333972930908,
+      "learning_rate": 4.151096669374492e-06,
+      "loss": 1.9204,
+      "step": 706
+    },
+    {
+      "epoch": 0.7747945205479452,
+      "grad_norm": 2.0357553958892822,
+      "learning_rate": 4.1490658001624705e-06,
+      "loss": 1.9399,
+      "step": 707
+    },
+    {
+      "epoch": 0.7758904109589041,
+      "grad_norm": 2.233694553375244,
+      "learning_rate": 4.147034930950447e-06,
+      "loss": 1.9182,
+      "step": 708
+    },
+    {
+      "epoch": 0.776986301369863,
+      "grad_norm": 2.1391239166259766,
+      "learning_rate": 4.145004061738425e-06,
+      "loss": 1.9157,
+      "step": 709
+    },
+    {
+      "epoch": 0.7780821917808219,
+      "grad_norm": 2.162320613861084,
+      "learning_rate": 4.1429731925264015e-06,
+      "loss": 1.9115,
+      "step": 710
+    },
+    {
+      "epoch": 0.7780821917808219,
+      "eval_loss": 2.0954039096832275,
+      "eval_runtime": 36.9862,
+      "eval_samples_per_second": 27.037,
+      "eval_steps_per_second": 1.136,
+      "step": 710
+    },
+    {
+      "epoch": 0.7791780821917809,
+      "grad_norm": 2.2382194995880127,
+      "learning_rate": 4.140942323314379e-06,
+      "loss": 1.8428,
+      "step": 711
+    },
+    {
+      "epoch": 0.7802739726027397,
+      "grad_norm": 2.3286185264587402,
+      "learning_rate": 4.138911454102356e-06,
+      "loss": 1.8906,
+      "step": 712
+    },
+    {
+      "epoch": 0.7813698630136986,
+      "grad_norm": 2.182708501815796,
+      "learning_rate": 4.136880584890333e-06,
+      "loss": 1.9132,
+      "step": 713
+    },
+    {
+      "epoch": 0.7824657534246575,
+      "grad_norm": 1.9978337287902832,
+      "learning_rate": 4.1348497156783106e-06,
+      "loss": 1.9433,
+      "step": 714
+    },
+    {
+      "epoch": 0.7835616438356164,
+      "grad_norm": 2.3092565536499023,
+      "learning_rate": 4.132818846466288e-06,
+      "loss": 1.9374,
+      "step": 715
+    },
+    {
+      "epoch": 0.7846575342465754,
+      "grad_norm": 2.0351054668426514,
+      "learning_rate": 4.130787977254265e-06,
+      "loss": 1.9361,
+      "step": 716
+    },
+    {
+      "epoch": 0.7857534246575343,
+      "grad_norm": 2.14770770072937,
+      "learning_rate": 4.128757108042242e-06,
+      "loss": 1.9553,
+      "step": 717
+    },
+    {
+      "epoch": 0.7868493150684932,
+      "grad_norm": 2.0613555908203125,
+      "learning_rate": 4.12672623883022e-06,
+      "loss": 1.9506,
+      "step": 718
+    },
+    {
+      "epoch": 0.787945205479452,
+      "grad_norm": 2.1312081813812256,
+      "learning_rate": 4.124695369618197e-06,
+      "loss": 2.0079,
+      "step": 719
+    },
+    {
+      "epoch": 0.7890410958904109,
+      "grad_norm": 2.088552474975586,
+      "learning_rate": 4.122664500406174e-06,
+      "loss": 1.9166,
+      "step": 720
+    },
+    {
+      "epoch": 0.7890410958904109,
+      "eval_loss": 2.093032121658325,
+      "eval_runtime": 36.8821,
+      "eval_samples_per_second": 27.113,
+      "eval_steps_per_second": 1.139,
+      "step": 720
+    },
+    {
+      "epoch": 0.7901369863013699,
+      "grad_norm": 2.1748929023742676,
+      "learning_rate": 4.1206336311941515e-06,
+      "loss": 1.9212,
+      "step": 721
+    },
+    {
+      "epoch": 0.7912328767123288,
+      "grad_norm": 2.1492016315460205,
+      "learning_rate": 4.118602761982129e-06,
+      "loss": 1.9452,
+      "step": 722
+    },
+    {
+      "epoch": 0.7923287671232877,
+      "grad_norm": 2.3364970684051514,
+      "learning_rate": 4.116571892770106e-06,
+      "loss": 1.8821,
+      "step": 723
+    },
+    {
+      "epoch": 0.7934246575342466,
+      "grad_norm": 2.318488597869873,
+      "learning_rate": 4.114541023558083e-06,
+      "loss": 1.8597,
+      "step": 724
+    },
+    {
+      "epoch": 0.7945205479452054,
+      "grad_norm": 2.153841257095337,
+      "learning_rate": 4.112510154346061e-06,
+      "loss": 1.9843,
+      "step": 725
+    },
+    {
+      "epoch": 0.7956164383561644,
+      "grad_norm": 2.3464651107788086,
+      "learning_rate": 4.110479285134038e-06,
+      "loss": 1.9105,
+      "step": 726
+    },
+    {
+      "epoch": 0.7967123287671233,
+      "grad_norm": 2.34413743019104,
+      "learning_rate": 4.108448415922015e-06,
+      "loss": 1.9285,
+      "step": 727
+    },
+    {
+      "epoch": 0.7978082191780822,
+      "grad_norm": 2.0822196006774902,
+      "learning_rate": 4.1064175467099924e-06,
+      "loss": 1.9536,
+      "step": 728
+    },
+    {
+      "epoch": 0.7989041095890411,
+      "grad_norm": 2.1978065967559814,
+      "learning_rate": 4.10438667749797e-06,
+      "loss": 1.9487,
+      "step": 729
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 2.4218783378601074,
+      "learning_rate": 4.102355808285947e-06,
+      "loss": 1.8113,
+      "step": 730
+    },
+    {
+      "epoch": 0.8,
+      "eval_loss": 2.0963101387023926,
+      "eval_runtime": 36.8853,
+      "eval_samples_per_second": 27.111,
+      "eval_steps_per_second": 1.139,
+      "step": 730
+    },
+    {
+      "epoch": 0.8010958904109589,
+      "grad_norm": 2.3887739181518555,
+      "learning_rate": 4.100324939073924e-06,
+      "loss": 2.0024,
+      "step": 731
+    },
+    {
+      "epoch": 0.8021917808219178,
+      "grad_norm": 2.296579360961914,
+      "learning_rate": 4.0982940698619015e-06,
+      "loss": 1.9092,
+      "step": 732
+    },
+    {
+      "epoch": 0.8032876712328767,
+      "grad_norm": 2.3881773948669434,
+      "learning_rate": 4.096263200649879e-06,
+      "loss": 1.9112,
+      "step": 733
+    },
+    {
+      "epoch": 0.8043835616438356,
+      "grad_norm": 2.40120530128479,
+      "learning_rate": 4.094232331437856e-06,
+      "loss": 1.9191,
+      "step": 734
+    },
+    {
+      "epoch": 0.8054794520547945,
+      "grad_norm": 2.184706211090088,
+      "learning_rate": 4.092201462225833e-06,
+      "loss": 1.8857,
+      "step": 735
+    },
+    {
+      "epoch": 0.8065753424657535,
+      "grad_norm": 2.3338732719421387,
+      "learning_rate": 4.090170593013811e-06,
+      "loss": 1.8889,
+      "step": 736
+    },
+    {
+      "epoch": 0.8076712328767123,
+      "grad_norm": 2.4065468311309814,
+      "learning_rate": 4.088139723801787e-06,
+      "loss": 1.9309,
+      "step": 737
+    },
+    {
+      "epoch": 0.8087671232876712,
+      "grad_norm": 2.044992446899414,
+      "learning_rate": 4.086108854589765e-06,
+      "loss": 1.8727,
+      "step": 738
+    },
+    {
+      "epoch": 0.8098630136986301,
+      "grad_norm": 2.23860239982605,
+      "learning_rate": 4.084077985377742e-06,
+      "loss": 1.8936,
+      "step": 739
+    },
+    {
+      "epoch": 0.810958904109589,
+      "grad_norm": 2.1324410438537598,
+      "learning_rate": 4.08204711616572e-06,
+      "loss": 1.9035,
+      "step": 740
+    },
+    {
+      "epoch": 0.810958904109589,
+      "eval_loss": 2.092885732650757,
+      "eval_runtime": 36.7991,
+      "eval_samples_per_second": 27.175,
+      "eval_steps_per_second": 1.141,
+      "step": 740
+    },
+    {
+      "epoch": 0.812054794520548,
+      "grad_norm": 2.337340831756592,
+      "learning_rate": 4.080016246953696e-06,
+      "loss": 1.9747,
+      "step": 741
+    },
+    {
+      "epoch": 0.8131506849315069,
+      "grad_norm": 2.262801170349121,
+      "learning_rate": 4.077985377741674e-06,
+      "loss": 1.8817,
+      "step": 742
+    },
+    {
+      "epoch": 0.8142465753424658,
+      "grad_norm": 2.0494258403778076,
+      "learning_rate": 4.075954508529651e-06,
+      "loss": 1.9493,
+      "step": 743
+    },
+    {
+      "epoch": 0.8153424657534246,
+      "grad_norm": 2.085829019546509,
+      "learning_rate": 4.073923639317629e-06,
+      "loss": 1.9611,
+      "step": 744
+    },
+    {
+      "epoch": 0.8164383561643835,
+      "grad_norm": 2.105806827545166,
+      "learning_rate": 4.071892770105605e-06,
+      "loss": 1.9779,
+      "step": 745
+    },
+    {
+      "epoch": 0.8175342465753425,
+      "grad_norm": 2.2664732933044434,
+      "learning_rate": 4.069861900893583e-06,
+      "loss": 1.9009,
+      "step": 746
+    },
+    {
+      "epoch": 0.8186301369863014,
+      "grad_norm": 1.9899768829345703,
+      "learning_rate": 4.06783103168156e-06,
+      "loss": 1.8585,
+      "step": 747
+    },
+    {
+      "epoch": 0.8197260273972603,
+      "grad_norm": 2.242236614227295,
+      "learning_rate": 4.065800162469538e-06,
+      "loss": 1.8904,
+      "step": 748
+    },
+    {
+      "epoch": 0.8208219178082192,
+      "grad_norm": 2.4481139183044434,
+      "learning_rate": 4.063769293257514e-06,
+      "loss": 1.8922,
+      "step": 749
+    },
+    {
+      "epoch": 0.821917808219178,
+      "grad_norm": 2.231722354888916,
+      "learning_rate": 4.061738424045492e-06,
+      "loss": 1.9292,
+      "step": 750
+    },
+    {
+      "epoch": 0.821917808219178,
+      "eval_loss": 2.0909852981567383,
+      "eval_runtime": 36.8255,
+      "eval_samples_per_second": 27.155,
+      "eval_steps_per_second": 1.141,
+      "step": 750
+    },
+    {
+      "epoch": 0.823013698630137,
+      "grad_norm": 2.323070764541626,
+      "learning_rate": 4.059707554833469e-06,
+      "loss": 1.9053,
+      "step": 751
+    },
+    {
+      "epoch": 0.8241095890410959,
+      "grad_norm": 2.137535572052002,
+      "learning_rate": 4.057676685621446e-06,
+      "loss": 1.9272,
+      "step": 752
+    },
+    {
+      "epoch": 0.8252054794520548,
+      "grad_norm": 2.2087643146514893,
+      "learning_rate": 4.0556458164094235e-06,
+      "loss": 1.9541,
+      "step": 753
+    },
+    {
+      "epoch": 0.8263013698630137,
+      "grad_norm": 2.04811692237854,
+      "learning_rate": 4.053614947197401e-06,
+      "loss": 1.922,
+      "step": 754
+    },
+    {
+      "epoch": 0.8273972602739726,
+      "grad_norm": 2.062622308731079,
+      "learning_rate": 4.051584077985378e-06,
+      "loss": 1.8388,
+      "step": 755
+    },
+    {
+      "epoch": 0.8284931506849315,
+      "grad_norm": 2.14631986618042,
+      "learning_rate": 4.049553208773355e-06,
+      "loss": 1.9153,
+      "step": 756
+    },
+    {
+      "epoch": 0.8295890410958904,
+      "grad_norm": 2.353529214859009,
+      "learning_rate": 4.047522339561333e-06,
+      "loss": 1.9623,
+      "step": 757
+    },
+    {
+      "epoch": 0.8306849315068493,
+      "grad_norm": 2.122342348098755,
+      "learning_rate": 4.04549147034931e-06,
+      "loss": 1.9095,
+      "step": 758
+    },
+    {
+      "epoch": 0.8317808219178082,
+      "grad_norm": 2.2540230751037598,
+      "learning_rate": 4.043460601137287e-06,
+      "loss": 1.9068,
+      "step": 759
+    },
+    {
+      "epoch": 0.8328767123287671,
+      "grad_norm": 2.164384365081787,
+      "learning_rate": 4.041429731925264e-06,
+      "loss": 2.0239,
+      "step": 760
+    },
+    {
+      "epoch": 0.8328767123287671,
+      "eval_loss": 2.0904648303985596,
+      "eval_runtime": 36.7617,
+      "eval_samples_per_second": 27.202,
+      "eval_steps_per_second": 1.142,
+      "step": 760
+    },
+    {
+      "epoch": 0.8339726027397261,
+      "grad_norm": 2.2487905025482178,
+      "learning_rate": 4.039398862713242e-06,
+      "loss": 1.9109,
+      "step": 761
+    },
+    {
+      "epoch": 0.8350684931506849,
+      "grad_norm": 2.0947930812835693,
+      "learning_rate": 4.037367993501219e-06,
+      "loss": 1.9133,
+      "step": 762
+    },
+    {
+      "epoch": 0.8361643835616438,
+      "grad_norm": 2.2259862422943115,
+      "learning_rate": 4.035337124289196e-06,
+      "loss": 1.9821,
+      "step": 763
+    },
+    {
+      "epoch": 0.8372602739726027,
+      "grad_norm": 2.0616455078125,
+      "learning_rate": 4.0333062550771735e-06,
+      "loss": 1.9037,
+      "step": 764
+    },
+    {
+      "epoch": 0.8383561643835616,
+      "grad_norm": 2.2704474925994873,
+      "learning_rate": 4.031275385865151e-06,
+      "loss": 1.8664,
+      "step": 765
+    },
+    {
+      "epoch": 0.8394520547945206,
+      "grad_norm": 2.0059332847595215,
+      "learning_rate": 4.029244516653128e-06,
+      "loss": 1.908,
+      "step": 766
+    },
+    {
+      "epoch": 0.8405479452054795,
+      "grad_norm": 2.0462095737457275,
+      "learning_rate": 4.027213647441105e-06,
+      "loss": 1.9397,
+      "step": 767
+    },
+    {
+      "epoch": 0.8416438356164384,
+      "grad_norm": 2.150207996368408,
+      "learning_rate": 4.025182778229083e-06,
+      "loss": 1.8505,
+      "step": 768
+    },
+    {
+      "epoch": 0.8427397260273972,
+      "grad_norm": 2.174863815307617,
+      "learning_rate": 4.02315190901706e-06,
+      "loss": 1.8895,
+      "step": 769
+    },
+    {
+      "epoch": 0.8438356164383561,
+      "grad_norm": 2.1907355785369873,
+      "learning_rate": 4.021121039805037e-06,
+      "loss": 1.9156,
+      "step": 770
+    },
+    {
+      "epoch": 0.8438356164383561,
+      "eval_loss": 2.0900955200195312,
+      "eval_runtime": 36.8004,
+      "eval_samples_per_second": 27.174,
+      "eval_steps_per_second": 1.141,
+      "step": 770
+    },
+    {
+      "epoch": 0.8449315068493151,
+      "grad_norm": 2.2831835746765137,
+      "learning_rate": 4.0190901705930144e-06,
+      "loss": 1.9115,
+      "step": 771
+    },
+    {
+      "epoch": 0.846027397260274,
+      "grad_norm": 2.039520263671875,
+      "learning_rate": 4.017059301380992e-06,
+      "loss": 2.0047,
+      "step": 772
+    },
+    {
+      "epoch": 0.8471232876712329,
+      "grad_norm": 2.147209405899048,
+      "learning_rate": 4.015028432168969e-06,
+      "loss": 1.8845,
+      "step": 773
+    },
+    {
+      "epoch": 0.8482191780821918,
+      "grad_norm": 2.168997287750244,
+      "learning_rate": 4.012997562956945e-06,
+      "loss": 1.9606,
+      "step": 774
+    },
+    {
+      "epoch": 0.8493150684931506,
+      "grad_norm": 2.038966417312622,
+      "learning_rate": 4.0109666937449235e-06,
+      "loss": 1.8734,
+      "step": 775
+    },
+    {
+      "epoch": 0.8504109589041096,
+      "grad_norm": 2.0531768798828125,
+      "learning_rate": 4.0089358245329e-06,
+      "loss": 1.889,
+      "step": 776
+    },
+    {
+      "epoch": 0.8515068493150685,
+      "grad_norm": 2.348008155822754,
+      "learning_rate": 4.006904955320878e-06,
+      "loss": 1.8972,
+      "step": 777
+    },
+    {
+      "epoch": 0.8526027397260274,
+      "grad_norm": 2.1647253036499023,
+      "learning_rate": 4.0048740861088545e-06,
+      "loss": 1.8652,
+      "step": 778
+    },
+    {
+      "epoch": 0.8536986301369863,
+      "grad_norm": 2.1971640586853027,
+      "learning_rate": 4.002843216896833e-06,
+      "loss": 1.9495,
+      "step": 779
+    },
+    {
+      "epoch": 0.8547945205479452,
+      "grad_norm": 2.169632911682129,
+      "learning_rate": 4.000812347684809e-06,
+      "loss": 1.9307,
+      "step": 780
+    },
+    {
+      "epoch": 0.8547945205479452,
+      "eval_loss": 2.087860107421875,
+      "eval_runtime": 36.7982,
+      "eval_samples_per_second": 27.175,
+      "eval_steps_per_second": 1.141,
+      "step": 780
+    },
+    {
+      "epoch": 0.8558904109589041,
+      "grad_norm": 2.4424400329589844,
+      "learning_rate": 3.998781478472787e-06,
+      "loss": 1.8849,
+      "step": 781
+    },
+    {
+      "epoch": 0.856986301369863,
+      "grad_norm": 2.2226672172546387,
+      "learning_rate": 3.996750609260764e-06,
+      "loss": 1.8975,
+      "step": 782
+    },
+    {
+      "epoch": 0.8580821917808219,
+      "grad_norm": 2.1898491382598877,
+      "learning_rate": 3.994719740048742e-06,
+      "loss": 1.8751,
+      "step": 783
+    },
+    {
+      "epoch": 0.8591780821917808,
+      "grad_norm": 2.466963052749634,
+      "learning_rate": 3.992688870836718e-06,
+      "loss": 2.0155,
+      "step": 784
+    },
+    {
+      "epoch": 0.8602739726027397,
+      "grad_norm": 2.184039831161499,
+      "learning_rate": 3.990658001624696e-06,
+      "loss": 1.9133,
+      "step": 785
+    },
+    {
+      "epoch": 0.8613698630136987,
+      "grad_norm": 2.089228868484497,
+      "learning_rate": 3.988627132412673e-06,
+      "loss": 1.8707,
+      "step": 786
+    },
+    {
+      "epoch": 0.8624657534246576,
+      "grad_norm": 2.0700666904449463,
+      "learning_rate": 3.98659626320065e-06,
+      "loss": 1.9487,
+      "step": 787
+    },
+    {
+      "epoch": 0.8635616438356164,
+      "grad_norm": 2.1495554447174072,
+      "learning_rate": 3.984565393988627e-06,
+      "loss": 1.9037,
+      "step": 788
+    },
+    {
+      "epoch": 0.8646575342465753,
+      "grad_norm": 2.0916988849639893,
+      "learning_rate": 3.9825345247766046e-06,
+      "loss": 1.9096,
+      "step": 789
+    },
+    {
+      "epoch": 0.8657534246575342,
+      "grad_norm": 2.072387933731079,
+      "learning_rate": 3.980503655564582e-06,
+      "loss": 1.9139,
+      "step": 790
+    },
+    {
+      "epoch": 0.8657534246575342,
+      "eval_loss": 2.0876142978668213,
+      "eval_runtime": 37.1068,
+      "eval_samples_per_second": 26.949,
+      "eval_steps_per_second": 1.132,
+      "step": 790
+    },
+    {
+      "epoch": 0.8668493150684932,
+      "grad_norm": 2.1206343173980713,
+      "learning_rate": 3.978472786352559e-06,
+      "loss": 1.9331,
+      "step": 791
+    },
+    {
+      "epoch": 0.8679452054794521,
+      "grad_norm": 2.20263934135437,
+      "learning_rate": 3.976441917140536e-06,
+      "loss": 1.9297,
+      "step": 792
+    },
+    {
+      "epoch": 0.869041095890411,
+      "grad_norm": 2.027554750442505,
+      "learning_rate": 3.974411047928514e-06,
+      "loss": 1.919,
+      "step": 793
+    },
+    {
+      "epoch": 0.8701369863013698,
+      "grad_norm": 2.0291616916656494,
+      "learning_rate": 3.972380178716491e-06,
+      "loss": 1.9682,
+      "step": 794
+    },
+    {
+      "epoch": 0.8712328767123287,
+      "grad_norm": 2.2719762325286865,
+      "learning_rate": 3.970349309504468e-06,
+      "loss": 1.9407,
+      "step": 795
+    },
+    {
+      "epoch": 0.8723287671232877,
+      "grad_norm": 2.0452375411987305,
+      "learning_rate": 3.9683184402924455e-06,
+      "loss": 1.9487,
+      "step": 796
+    },
+    {
+      "epoch": 0.8734246575342466,
+      "grad_norm": 2.170441150665283,
+      "learning_rate": 3.966287571080423e-06,
+      "loss": 1.8651,
+      "step": 797
+    },
+    {
+      "epoch": 0.8745205479452055,
+      "grad_norm": 2.3370516300201416,
+      "learning_rate": 3.9642567018684e-06,
+      "loss": 1.8599,
+      "step": 798
+    },
+    {
+      "epoch": 0.8756164383561644,
+      "grad_norm": 2.336737632751465,
+      "learning_rate": 3.962225832656377e-06,
+      "loss": 1.9304,
+      "step": 799
+    },
+    {
+      "epoch": 0.8767123287671232,
+      "grad_norm": 1.9708369970321655,
+      "learning_rate": 3.960194963444355e-06,
+      "loss": 1.9125,
+      "step": 800
+    },
+    {
+      "epoch": 0.8767123287671232,
+      "eval_loss": 2.0812273025512695,
+      "eval_runtime": 37.0752,
+      "eval_samples_per_second": 26.972,
+      "eval_steps_per_second": 1.133,
+      "step": 800
+    },
+    {
+      "epoch": 0.8778082191780822,
+      "grad_norm": 2.154219627380371,
+      "learning_rate": 3.958164094232332e-06,
+      "loss": 1.8304,
+      "step": 801
+    },
+    {
+      "epoch": 0.8789041095890411,
+      "grad_norm": 2.237786054611206,
+      "learning_rate": 3.956133225020309e-06,
+      "loss": 1.9147,
+      "step": 802
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 2.07143235206604,
+      "learning_rate": 3.954102355808286e-06,
+      "loss": 1.9128,
+      "step": 803
+    },
+    {
+      "epoch": 0.8810958904109589,
+      "grad_norm": 2.2780392169952393,
+      "learning_rate": 3.952071486596264e-06,
+      "loss": 1.9145,
+      "step": 804
+    },
+    {
+      "epoch": 0.8821917808219178,
+      "grad_norm": 2.2396292686462402,
+      "learning_rate": 3.950040617384241e-06,
+      "loss": 1.9049,
+      "step": 805
+    },
+    {
+      "epoch": 0.8832876712328767,
+      "grad_norm": 2.426511287689209,
+      "learning_rate": 3.948009748172218e-06,
+      "loss": 1.8984,
+      "step": 806
+    },
+    {
+      "epoch": 0.8843835616438356,
+      "grad_norm": 2.1874325275421143,
+      "learning_rate": 3.9459788789601955e-06,
+      "loss": 1.8888,
+      "step": 807
+    },
+    {
+      "epoch": 0.8854794520547945,
+      "grad_norm": 2.038098096847534,
+      "learning_rate": 3.943948009748173e-06,
+      "loss": 1.9582,
+      "step": 808
+    },
+    {
+      "epoch": 0.8865753424657534,
+      "grad_norm": 2.3748130798339844,
+      "learning_rate": 3.94191714053615e-06,
+      "loss": 1.9091,
+      "step": 809
+    },
+    {
+      "epoch": 0.8876712328767123,
+      "grad_norm": 2.2457494735717773,
+      "learning_rate": 3.939886271324127e-06,
+      "loss": 1.9067,
+      "step": 810
+    },
+    {
+      "epoch": 0.8876712328767123,
+      "eval_loss": 2.0766079425811768,
+      "eval_runtime": 36.9896,
+      "eval_samples_per_second": 27.035,
+      "eval_steps_per_second": 1.135,
+      "step": 810
+    },
+    {
+      "epoch": 0.8887671232876713,
+      "grad_norm": 2.156644821166992,
+      "learning_rate": 3.937855402112104e-06,
+      "loss": 1.9347,
+      "step": 811
+    },
+    {
+      "epoch": 0.8898630136986302,
+      "grad_norm": 2.3991472721099854,
+      "learning_rate": 3.935824532900082e-06,
+      "loss": 1.8645,
+      "step": 812
+    },
+    {
+      "epoch": 0.890958904109589,
+      "grad_norm": 2.252729654312134,
+      "learning_rate": 3.933793663688058e-06,
+      "loss": 1.8537,
+      "step": 813
+    },
+    {
+      "epoch": 0.8920547945205479,
+      "grad_norm": 2.2917964458465576,
+      "learning_rate": 3.9317627944760365e-06,
+      "loss": 1.9271,
+      "step": 814
+    },
+    {
+      "epoch": 0.8931506849315068,
+      "grad_norm": 2.217310667037964,
+      "learning_rate": 3.929731925264013e-06,
+      "loss": 1.9406,
+      "step": 815
+    },
+    {
+      "epoch": 0.8942465753424658,
+      "grad_norm": 2.167931079864502,
+      "learning_rate": 3.927701056051991e-06,
+      "loss": 1.8554,
+      "step": 816
+    },
+    {
+      "epoch": 0.8953424657534247,
+      "grad_norm": 2.18337082862854,
+      "learning_rate": 3.9256701868399674e-06,
+      "loss": 1.8795,
+      "step": 817
+    },
+    {
+      "epoch": 0.8964383561643836,
+      "grad_norm": 2.2348973751068115,
+      "learning_rate": 3.9236393176279456e-06,
+      "loss": 1.8831,
+      "step": 818
+    },
+    {
+      "epoch": 0.8975342465753424,
+      "grad_norm": 2.1822657585144043,
+      "learning_rate": 3.921608448415922e-06,
+      "loss": 1.8728,
+      "step": 819
+    },
+    {
+      "epoch": 0.8986301369863013,
+      "grad_norm": 2.5253243446350098,
+      "learning_rate": 3.9195775792039e-06,
+      "loss": 1.8633,
+      "step": 820
+    },
+    {
+      "epoch": 0.8986301369863013,
+      "eval_loss": 2.0769834518432617,
+      "eval_runtime": 36.838,
+      "eval_samples_per_second": 27.146,
+      "eval_steps_per_second": 1.14,
+      "step": 820
+    },
+    {
+      "epoch": 0.8997260273972603,
+      "grad_norm": 2.2109973430633545,
+      "learning_rate": 3.9175467099918765e-06,
+      "loss": 1.912,
+      "step": 821
+    },
+    {
+      "epoch": 0.9008219178082192,
+      "grad_norm": 2.053692102432251,
+      "learning_rate": 3.915515840779855e-06,
+      "loss": 1.8928,
+      "step": 822
+    },
+    {
+      "epoch": 0.9019178082191781,
+      "grad_norm": 2.1418213844299316,
+      "learning_rate": 3.913484971567831e-06,
+      "loss": 1.9399,
+      "step": 823
+    },
+    {
+      "epoch": 0.903013698630137,
+      "grad_norm": 2.3615458011627197,
+      "learning_rate": 3.911454102355808e-06,
+      "loss": 1.9293,
+      "step": 824
+    },
+    {
+      "epoch": 0.9041095890410958,
+      "grad_norm": 2.098254680633545,
+      "learning_rate": 3.909423233143786e-06,
+      "loss": 1.8301,
+      "step": 825
+    },
+    {
+      "epoch": 0.9052054794520548,
+      "grad_norm": 2.4473679065704346,
+      "learning_rate": 3.907392363931763e-06,
+      "loss": 1.9254,
+      "step": 826
+    },
+    {
+      "epoch": 0.9063013698630137,
+      "grad_norm": 2.252843141555786,
+      "learning_rate": 3.90536149471974e-06,
+      "loss": 1.9155,
+      "step": 827
+    },
+    {
+      "epoch": 0.9073972602739726,
+      "grad_norm": 2.1574337482452393,
+      "learning_rate": 3.9033306255077175e-06,
+      "loss": 1.9809,
+      "step": 828
+    },
+    {
+      "epoch": 0.9084931506849315,
+      "grad_norm": 2.2899720668792725,
+      "learning_rate": 3.901299756295695e-06,
+      "loss": 2.0045,
+      "step": 829
+    },
+    {
+      "epoch": 0.9095890410958904,
+      "grad_norm": 2.227285146713257,
+      "learning_rate": 3.899268887083672e-06,
+      "loss": 2.026,
+      "step": 830
+    },
+    {
+      "epoch": 0.9095890410958904,
+      "eval_loss": 2.0764760971069336,
+      "eval_runtime": 36.9195,
+      "eval_samples_per_second": 27.086,
+      "eval_steps_per_second": 1.138,
+      "step": 830
+    },
+    {
+      "epoch": 0.9106849315068493,
+      "grad_norm": 2.1935312747955322,
+      "learning_rate": 3.897238017871649e-06,
+      "loss": 1.8922,
+      "step": 831
+    },
+    {
+      "epoch": 0.9117808219178082,
+      "grad_norm": 2.129692316055298,
+      "learning_rate": 3.8952071486596266e-06,
+      "loss": 1.8981,
+      "step": 832
+    },
+    {
+      "epoch": 0.9128767123287671,
+      "grad_norm": 2.4435646533966064,
+      "learning_rate": 3.893176279447604e-06,
+      "loss": 1.91,
+      "step": 833
+    },
+    {
+      "epoch": 0.913972602739726,
+      "grad_norm": 2.1556732654571533,
+      "learning_rate": 3.891145410235581e-06,
+      "loss": 1.8688,
+      "step": 834
+    },
+    {
+      "epoch": 0.915068493150685,
+      "grad_norm": 2.175891160964966,
+      "learning_rate": 3.889114541023558e-06,
+      "loss": 1.9777,
+      "step": 835
+    },
+    {
+      "epoch": 0.9161643835616439,
+      "grad_norm": 2.220466375350952,
+      "learning_rate": 3.887083671811536e-06,
+      "loss": 1.9152,
+      "step": 836
+    },
+    {
+      "epoch": 0.9172602739726028,
+      "grad_norm": 2.331397294998169,
+      "learning_rate": 3.885052802599513e-06,
+      "loss": 1.892,
+      "step": 837
+    },
+    {
+      "epoch": 0.9183561643835616,
+      "grad_norm": 2.2434773445129395,
+      "learning_rate": 3.88302193338749e-06,
+      "loss": 1.841,
+      "step": 838
+    },
+    {
+      "epoch": 0.9194520547945205,
+      "grad_norm": 2.3248002529144287,
+      "learning_rate": 3.8809910641754675e-06,
+      "loss": 1.9657,
+      "step": 839
+    },
+    {
+      "epoch": 0.9205479452054794,
+      "grad_norm": 2.090252637863159,
+      "learning_rate": 3.878960194963445e-06,
+      "loss": 1.9121,
+      "step": 840
+    },
+    {
+      "epoch": 0.9205479452054794,
+      "eval_loss": 2.0798065662384033,
+      "eval_runtime": 36.8557,
+      "eval_samples_per_second": 27.133,
+      "eval_steps_per_second": 1.14,
+      "step": 840
+    },
+    {
+      "epoch": 0.9216438356164384,
+      "grad_norm": 2.6968672275543213,
+      "learning_rate": 3.876929325751422e-06,
+      "loss": 1.9003,
+      "step": 841
+    },
+    {
+      "epoch": 0.9227397260273973,
+      "grad_norm": 2.356321096420288,
+      "learning_rate": 3.874898456539399e-06,
+      "loss": 1.9471,
+      "step": 842
+    },
+    {
+      "epoch": 0.9238356164383562,
+      "grad_norm": 2.2678606510162354,
+      "learning_rate": 3.872867587327377e-06,
+      "loss": 1.8488,
+      "step": 843
+    },
+    {
+      "epoch": 0.924931506849315,
+      "grad_norm": 2.1122310161590576,
+      "learning_rate": 3.870836718115354e-06,
+      "loss": 1.9247,
+      "step": 844
+    },
+    {
+      "epoch": 0.9260273972602739,
+      "grad_norm": 2.1219136714935303,
+      "learning_rate": 3.868805848903331e-06,
+      "loss": 1.8829,
+      "step": 845
+    },
+    {
+      "epoch": 0.9271232876712329,
+      "grad_norm": 2.175607204437256,
+      "learning_rate": 3.8667749796913084e-06,
+      "loss": 1.887,
+      "step": 846
+    },
+    {
+      "epoch": 0.9282191780821918,
+      "grad_norm": 2.156214714050293,
+      "learning_rate": 3.864744110479286e-06,
+      "loss": 1.9119,
+      "step": 847
+    },
+    {
+      "epoch": 0.9293150684931507,
+      "grad_norm": 2.158461570739746,
+      "learning_rate": 3.862713241267262e-06,
+      "loss": 1.9092,
+      "step": 848
+    },
+    {
+      "epoch": 0.9304109589041096,
+      "grad_norm": 2.180551767349243,
+      "learning_rate": 3.86068237205524e-06,
+      "loss": 1.946,
+      "step": 849
+    },
+    {
+      "epoch": 0.9315068493150684,
+      "grad_norm": 2.1030542850494385,
+      "learning_rate": 3.858651502843217e-06,
+      "loss": 1.9484,
+      "step": 850
+    },
+    {
+      "epoch": 0.9315068493150684,
+      "eval_loss": 2.0767791271209717,
+      "eval_runtime": 36.8047,
+      "eval_samples_per_second": 27.17,
+      "eval_steps_per_second": 1.141,
+      "step": 850
+    },
+    {
+      "epoch": 0.9326027397260274,
+      "grad_norm": 2.240769386291504,
+      "learning_rate": 3.856620633631195e-06,
+      "loss": 1.9114,
+      "step": 851
+    },
+    {
+      "epoch": 0.9336986301369863,
+      "grad_norm": 2.1964619159698486,
+      "learning_rate": 3.854589764419171e-06,
+      "loss": 1.8339,
+      "step": 852
+    },
+    {
+      "epoch": 0.9347945205479452,
+      "grad_norm": 2.2875845432281494,
+      "learning_rate": 3.852558895207149e-06,
+      "loss": 1.8942,
+      "step": 853
+    },
+    {
+      "epoch": 0.9358904109589041,
+      "grad_norm": 2.1462507247924805,
+      "learning_rate": 3.850528025995126e-06,
+      "loss": 1.9838,
+      "step": 854
+    },
+    {
+      "epoch": 0.936986301369863,
+      "grad_norm": 2.0982913970947266,
+      "learning_rate": 3.848497156783104e-06,
+      "loss": 1.9351,
+      "step": 855
+    },
+    {
+      "epoch": 0.938082191780822,
+      "grad_norm": 2.260263442993164,
+      "learning_rate": 3.84646628757108e-06,
+      "loss": 1.8745,
+      "step": 856
+    },
+    {
+      "epoch": 0.9391780821917808,
+      "grad_norm": 2.091778039932251,
+      "learning_rate": 3.8444354183590585e-06,
+      "loss": 1.8888,
+      "step": 857
+    },
+    {
+      "epoch": 0.9402739726027397,
+      "grad_norm": 2.054974317550659,
+      "learning_rate": 3.842404549147035e-06,
+      "loss": 1.8801,
+      "step": 858
+    },
+    {
+      "epoch": 0.9413698630136986,
+      "grad_norm": 2.079641580581665,
+      "learning_rate": 3.840373679935013e-06,
+      "loss": 1.8879,
+      "step": 859
+    },
+    {
+      "epoch": 0.9424657534246575,
+      "grad_norm": 2.3441810607910156,
+      "learning_rate": 3.8383428107229894e-06,
+      "loss": 1.8571,
+      "step": 860
+    },
+    {
+      "epoch": 0.9424657534246575,
+      "eval_loss": 2.0766854286193848,
+      "eval_runtime": 36.8414,
+      "eval_samples_per_second": 27.143,
+      "eval_steps_per_second": 1.14,
+      "step": 860
+    },
+    {
+      "epoch": 0.9435616438356165,
+      "grad_norm": 2.123061180114746,
+      "learning_rate": 3.836311941510967e-06,
+      "loss": 1.8922,
+      "step": 861
+    },
+    {
+      "epoch": 0.9446575342465754,
+      "grad_norm": 2.1573386192321777,
+      "learning_rate": 3.834281072298944e-06,
+      "loss": 1.8713,
+      "step": 862
+    },
+    {
+      "epoch": 0.9457534246575342,
+      "grad_norm": 2.225186824798584,
+      "learning_rate": 3.832250203086921e-06,
+      "loss": 1.925,
+      "step": 863
+    },
+    {
+      "epoch": 0.9468493150684931,
+      "grad_norm": 2.1539998054504395,
+      "learning_rate": 3.8302193338748985e-06,
+      "loss": 1.8721,
+      "step": 864
+    },
+    {
+      "epoch": 0.947945205479452,
+      "grad_norm": 2.075580358505249,
+      "learning_rate": 3.828188464662876e-06,
+      "loss": 1.9489,
+      "step": 865
+    },
+    {
+      "epoch": 0.949041095890411,
+      "grad_norm": 2.1789908409118652,
+      "learning_rate": 3.826157595450853e-06,
+      "loss": 1.9883,
+      "step": 866
+    },
+    {
+      "epoch": 0.9501369863013699,
+      "grad_norm": 2.0838615894317627,
+      "learning_rate": 3.82412672623883e-06,
+      "loss": 1.9149,
+      "step": 867
+    },
+    {
+      "epoch": 0.9512328767123288,
+      "grad_norm": 2.1321213245391846,
+      "learning_rate": 3.822095857026808e-06,
+      "loss": 1.8505,
+      "step": 868
+    },
+    {
+      "epoch": 0.9523287671232876,
+      "grad_norm": 2.1419479846954346,
+      "learning_rate": 3.820064987814785e-06,
+      "loss": 1.9226,
+      "step": 869
+    },
+    {
+      "epoch": 0.9534246575342465,
+      "grad_norm": 2.290163040161133,
+      "learning_rate": 3.818034118602762e-06,
+      "loss": 1.8737,
+      "step": 870
+    },
+    {
+      "epoch": 0.9534246575342465,
+      "eval_loss": 2.073598623275757,
+      "eval_runtime": 36.7896,
+      "eval_samples_per_second": 27.182,
+      "eval_steps_per_second": 1.142,
+      "step": 870
+    },
+    {
+      "epoch": 0.9545205479452055,
+      "grad_norm": 2.1951074600219727,
+      "learning_rate": 3.8160032493907395e-06,
+      "loss": 1.8699,
+      "step": 871
+    },
+    {
+      "epoch": 0.9556164383561644,
+      "grad_norm": 2.25960111618042,
+      "learning_rate": 3.813972380178717e-06,
+      "loss": 1.8813,
+      "step": 872
+    },
+    {
+      "epoch": 0.9567123287671233,
+      "grad_norm": 2.4376907348632812,
+      "learning_rate": 3.811941510966694e-06,
+      "loss": 1.8782,
+      "step": 873
+    },
+    {
+      "epoch": 0.9578082191780822,
+      "grad_norm": 1.980772614479065,
+      "learning_rate": 3.809910641754671e-06,
+      "loss": 1.9205,
+      "step": 874
+    },
+    {
+      "epoch": 0.958904109589041,
+      "grad_norm": 2.0699028968811035,
+      "learning_rate": 3.8078797725426486e-06,
+      "loss": 1.8674,
+      "step": 875
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 2.2765867710113525,
+      "learning_rate": 3.8058489033306254e-06,
+      "loss": 1.81,
+      "step": 876
+    },
+    {
+      "epoch": 0.9610958904109589,
+      "grad_norm": 2.0998799800872803,
+      "learning_rate": 3.803818034118603e-06,
+      "loss": 1.9061,
+      "step": 877
+    },
+    {
+      "epoch": 0.9621917808219178,
+      "grad_norm": 2.2362914085388184,
+      "learning_rate": 3.80178716490658e-06,
+      "loss": 1.8798,
+      "step": 878
+    },
+    {
+      "epoch": 0.9632876712328767,
+      "grad_norm": 2.0789835453033447,
+      "learning_rate": 3.7997562956945577e-06,
+      "loss": 1.9491,
+      "step": 879
+    },
+    {
+      "epoch": 0.9643835616438357,
+      "grad_norm": 2.0458688735961914,
+      "learning_rate": 3.7977254264825345e-06,
+      "loss": 1.931,
+      "step": 880
+    },
+    {
+      "epoch": 0.9643835616438357,
+      "eval_loss": 2.0672221183776855,
+      "eval_runtime": 36.8218,
+      "eval_samples_per_second": 27.158,
+      "eval_steps_per_second": 1.141,
+      "step": 880
+    },
+    {
+      "epoch": 0.9654794520547946,
+      "grad_norm": 2.361626625061035,
+      "learning_rate": 3.7956945572705122e-06,
+      "loss": 1.8815,
+      "step": 881
+    },
+    {
+      "epoch": 0.9665753424657534,
+      "grad_norm": 2.294160842895508,
+      "learning_rate": 3.793663688058489e-06,
+      "loss": 1.8649,
+      "step": 882
+    },
+    {
+      "epoch": 0.9676712328767123,
+      "grad_norm": 2.2242612838745117,
+      "learning_rate": 3.791632818846467e-06,
+      "loss": 1.9535,
+      "step": 883
+    },
+    {
+      "epoch": 0.9687671232876712,
+      "grad_norm": 2.220269203186035,
+      "learning_rate": 3.7896019496344436e-06,
+      "loss": 1.9506,
+      "step": 884
+    },
+    {
+      "epoch": 0.9698630136986301,
+      "grad_norm": 2.112518072128296,
+      "learning_rate": 3.787571080422421e-06,
+      "loss": 1.9487,
+      "step": 885
+    },
+    {
+      "epoch": 0.9709589041095891,
+      "grad_norm": 2.17695951461792,
+      "learning_rate": 3.785540211210398e-06,
+      "loss": 1.9467,
+      "step": 886
+    },
+    {
+      "epoch": 0.972054794520548,
+      "grad_norm": 2.0990567207336426,
+      "learning_rate": 3.7835093419983755e-06,
+      "loss": 1.9216,
+      "step": 887
+    },
+    {
+      "epoch": 0.9731506849315068,
+      "grad_norm": 2.222566604614258,
+      "learning_rate": 3.7814784727863527e-06,
+      "loss": 1.9043,
+      "step": 888
+    },
+    {
+      "epoch": 0.9742465753424657,
+      "grad_norm": 2.09890079498291,
+      "learning_rate": 3.77944760357433e-06,
+      "loss": 1.884,
+      "step": 889
+    },
+    {
+      "epoch": 0.9753424657534246,
+      "grad_norm": 2.136141061782837,
+      "learning_rate": 3.7774167343623073e-06,
+      "loss": 2.0032,
+      "step": 890
+    },
+    {
+      "epoch": 0.9753424657534246,
+      "eval_loss": 2.0665438175201416,
+      "eval_runtime": 37.1207,
+      "eval_samples_per_second": 26.939,
+      "eval_steps_per_second": 1.131,
+      "step": 890
+    },
+    {
+      "epoch": 0.9764383561643836,
+      "grad_norm": 2.267444372177124,
+      "learning_rate": 3.7753858651502846e-06,
+      "loss": 1.947,
+      "step": 891
+    },
+    {
+      "epoch": 0.9775342465753425,
+      "grad_norm": 2.1141815185546875,
+      "learning_rate": 3.773354995938262e-06,
+      "loss": 1.8705,
+      "step": 892
+    },
+    {
+      "epoch": 0.9786301369863014,
+      "grad_norm": 2.1036782264709473,
+      "learning_rate": 3.771324126726239e-06,
+      "loss": 1.9381,
+      "step": 893
+    },
+    {
+      "epoch": 0.9797260273972602,
+      "grad_norm": 2.081982135772705,
+      "learning_rate": 3.7692932575142164e-06,
+      "loss": 1.9079,
+      "step": 894
+    },
+    {
+      "epoch": 0.9808219178082191,
+      "grad_norm": 2.0815446376800537,
+      "learning_rate": 3.7672623883021937e-06,
+      "loss": 1.9417,
+      "step": 895
+    },
+    {
+      "epoch": 0.9819178082191781,
+      "grad_norm": 2.0531511306762695,
+      "learning_rate": 3.765231519090171e-06,
+      "loss": 1.876,
+      "step": 896
+    },
+    {
+      "epoch": 0.983013698630137,
+      "grad_norm": 2.212940216064453,
+      "learning_rate": 3.7632006498781482e-06,
+      "loss": 1.8008,
+      "step": 897
+    },
+    {
+      "epoch": 0.9841095890410959,
+      "grad_norm": 2.1541531085968018,
+      "learning_rate": 3.761169780666125e-06,
+      "loss": 1.9039,
+      "step": 898
+    },
+    {
+      "epoch": 0.9852054794520548,
+      "grad_norm": 2.113884925842285,
+      "learning_rate": 3.7591389114541028e-06,
+      "loss": 1.8562,
+      "step": 899
+    },
+    {
+      "epoch": 0.9863013698630136,
+      "grad_norm": 2.1503701210021973,
+      "learning_rate": 3.7571080422420796e-06,
+      "loss": 1.9052,
+      "step": 900
+    },
+    {
+      "epoch": 0.9863013698630136,
+      "eval_loss": 2.0664408206939697,
+      "eval_runtime": 37.0876,
+      "eval_samples_per_second": 26.963,
+      "eval_steps_per_second": 1.132,
+      "step": 900
+    },
+    {
+      "epoch": 0.9873972602739726,
+      "grad_norm": 2.1584348678588867,
+      "learning_rate": 3.7550771730300573e-06,
+      "loss": 1.9296,
+      "step": 901
+    },
+    {
+      "epoch": 0.9884931506849315,
+      "grad_norm": 2.164318084716797,
+      "learning_rate": 3.753046303818034e-06,
+      "loss": 1.8834,
+      "step": 902
+    },
+    {
+      "epoch": 0.9895890410958904,
+      "grad_norm": 2.0222678184509277,
+      "learning_rate": 3.751015434606012e-06,
+      "loss": 1.9426,
+      "step": 903
+    },
+    {
+      "epoch": 0.9906849315068493,
+      "grad_norm": 2.116835832595825,
+      "learning_rate": 3.7489845653939887e-06,
+      "loss": 1.9195,
+      "step": 904
+    },
+    {
+      "epoch": 0.9917808219178083,
+      "grad_norm": 2.2708873748779297,
+      "learning_rate": 3.7469536961819664e-06,
+      "loss": 1.9631,
+      "step": 905
+    },
+    {
+      "epoch": 0.9928767123287672,
+      "grad_norm": 2.1174001693725586,
+      "learning_rate": 3.7449228269699433e-06,
+      "loss": 1.8602,
+      "step": 906
+    },
+    {
+      "epoch": 0.993972602739726,
+      "grad_norm": 2.091830015182495,
+      "learning_rate": 3.742891957757921e-06,
+      "loss": 1.8906,
+      "step": 907
+    },
+    {
+      "epoch": 0.9950684931506849,
+      "grad_norm": 2.007119655609131,
+      "learning_rate": 3.740861088545898e-06,
+      "loss": 1.9122,
+      "step": 908
+    },
+    {
+      "epoch": 0.9961643835616438,
+      "grad_norm": 2.070396661758423,
+      "learning_rate": 3.7388302193338755e-06,
+      "loss": 1.9106,
+      "step": 909
+    },
+    {
+      "epoch": 0.9972602739726028,
+      "grad_norm": 2.080331325531006,
+      "learning_rate": 3.7367993501218524e-06,
+      "loss": 1.934,
+      "step": 910
+    },
+    {
+      "epoch": 0.9972602739726028,
+      "eval_loss": 2.0694408416748047,
+      "eval_runtime": 36.9953,
+      "eval_samples_per_second": 27.03,
+      "eval_steps_per_second": 1.135,
+      "step": 910
+    },
+    {
+      "epoch": 0.9983561643835617,
+      "grad_norm": 2.0920615196228027,
+      "learning_rate": 3.7347684809098297e-06,
+      "loss": 1.8679,
+      "step": 911
+    },
+    {
+      "epoch": 0.9994520547945206,
+      "grad_norm": 2.112732172012329,
+      "learning_rate": 3.732737611697807e-06,
+      "loss": 1.8723,
+      "step": 912
+    },
+    {
+      "epoch": 1.0005479452054795,
+      "grad_norm": 2.1449220180511475,
+      "learning_rate": 3.7307067424857842e-06,
+      "loss": 1.8509,
+      "step": 913
+    },
+    {
+      "epoch": 1.0016438356164383,
+      "grad_norm": 2.2179951667785645,
+      "learning_rate": 3.7286758732737615e-06,
+      "loss": 1.7911,
+      "step": 914
+    },
+    {
+      "epoch": 1.0027397260273974,
+      "grad_norm": 2.1392273902893066,
+      "learning_rate": 3.7266450040617388e-06,
+      "loss": 1.8475,
+      "step": 915
+    },
+    {
+      "epoch": 1.0038356164383562,
+      "grad_norm": 2.190265417098999,
+      "learning_rate": 3.724614134849716e-06,
+      "loss": 1.8394,
+      "step": 916
+    },
+    {
+      "epoch": 1.004931506849315,
+      "grad_norm": 2.1833760738372803,
+      "learning_rate": 3.7225832656376933e-06,
+      "loss": 1.9696,
+      "step": 917
+    },
+    {
+      "epoch": 1.006027397260274,
+      "grad_norm": 2.2504308223724365,
+      "learning_rate": 3.7205523964256706e-06,
+      "loss": 1.8135,
+      "step": 918
+    },
+    {
+      "epoch": 1.0071232876712328,
+      "grad_norm": 2.3909389972686768,
+      "learning_rate": 3.718521527213648e-06,
+      "loss": 1.9331,
+      "step": 919
+    },
+    {
+      "epoch": 1.0082191780821919,
+      "grad_norm": 2.3367602825164795,
+      "learning_rate": 3.716490658001625e-06,
+      "loss": 1.9026,
+      "step": 920
+    },
+    {
+      "epoch": 1.0082191780821919,
+      "eval_loss": 2.066087484359741,
+      "eval_runtime": 36.8701,
+      "eval_samples_per_second": 27.122,
+      "eval_steps_per_second": 1.139,
+      "step": 920
+    },
+    {
+      "epoch": 1.0093150684931507,
+      "grad_norm": 2.1732144355773926,
+      "learning_rate": 3.7144597887896024e-06,
+      "loss": 1.8888,
+      "step": 921
+    },
+    {
+      "epoch": 1.0104109589041095,
+      "grad_norm": 2.4632201194763184,
+      "learning_rate": 3.7124289195775797e-06,
+      "loss": 1.7725,
+      "step": 922
+    },
+    {
+      "epoch": 1.0115068493150685,
+      "grad_norm": 2.4887173175811768,
+      "learning_rate": 3.710398050365557e-06,
+      "loss": 1.9125,
+      "step": 923
+    },
+    {
+      "epoch": 1.0126027397260273,
+      "grad_norm": 2.4684619903564453,
+      "learning_rate": 3.708367181153534e-06,
+      "loss": 1.8829,
+      "step": 924
+    },
+    {
+      "epoch": 1.0136986301369864,
+      "grad_norm": 2.348788022994995,
+      "learning_rate": 3.7063363119415115e-06,
+      "loss": 1.8542,
+      "step": 925
+    },
+    {
+      "epoch": 1.0147945205479452,
+      "grad_norm": 2.4432766437530518,
+      "learning_rate": 3.7043054427294884e-06,
+      "loss": 1.903,
+      "step": 926
+    },
+    {
+      "epoch": 1.0158904109589042,
+      "grad_norm": 2.425910234451294,
+      "learning_rate": 3.702274573517466e-06,
+      "loss": 1.8532,
+      "step": 927
+    },
+    {
+      "epoch": 1.016986301369863,
+      "grad_norm": 2.447777032852173,
+      "learning_rate": 3.700243704305443e-06,
+      "loss": 1.8277,
+      "step": 928
+    },
+    {
+      "epoch": 1.0180821917808218,
+      "grad_norm": 2.1830408573150635,
+      "learning_rate": 3.6982128350934206e-06,
+      "loss": 1.8546,
+      "step": 929
+    },
+    {
+      "epoch": 1.0191780821917809,
+      "grad_norm": 2.2692818641662598,
+      "learning_rate": 3.6961819658813975e-06,
+      "loss": 1.8356,
+      "step": 930
+    },
+    {
+      "epoch": 1.0191780821917809,
+      "eval_loss": 2.0714614391326904,
+      "eval_runtime": 36.8661,
+      "eval_samples_per_second": 27.125,
+      "eval_steps_per_second": 1.139,
+      "step": 930
+    },
+    {
+      "epoch": 1.0202739726027397,
+      "grad_norm": 2.5044169425964355,
+      "learning_rate": 3.694151096669375e-06,
+      "loss": 1.8173,
+      "step": 931
+    },
+    {
+      "epoch": 1.0213698630136987,
+      "grad_norm": 2.419637680053711,
+      "learning_rate": 3.692120227457352e-06,
+      "loss": 1.9418,
+      "step": 932
+    },
+    {
+      "epoch": 1.0224657534246575,
+      "grad_norm": 2.282288074493408,
+      "learning_rate": 3.6900893582453297e-06,
+      "loss": 1.8698,
+      "step": 933
+    },
+    {
+      "epoch": 1.0235616438356165,
+      "grad_norm": 2.3215551376342773,
+      "learning_rate": 3.6880584890333066e-06,
+      "loss": 1.8935,
+      "step": 934
+    },
+    {
+      "epoch": 1.0246575342465754,
+      "grad_norm": 2.5938258171081543,
+      "learning_rate": 3.6860276198212834e-06,
+      "loss": 1.79,
+      "step": 935
+    },
+    {
+      "epoch": 1.0257534246575342,
+      "grad_norm": 2.215024471282959,
+      "learning_rate": 3.683996750609261e-06,
+      "loss": 1.8768,
+      "step": 936
+    },
+    {
+      "epoch": 1.0268493150684932,
+      "grad_norm": 2.239609956741333,
+      "learning_rate": 3.681965881397238e-06,
+      "loss": 1.8887,
+      "step": 937
+    },
+    {
+      "epoch": 1.027945205479452,
+      "grad_norm": 2.3134827613830566,
+      "learning_rate": 3.6799350121852157e-06,
+      "loss": 1.8897,
+      "step": 938
+    },
+    {
+      "epoch": 1.029041095890411,
+      "grad_norm": 2.4100260734558105,
+      "learning_rate": 3.6779041429731925e-06,
+      "loss": 1.909,
+      "step": 939
+    },
+    {
+      "epoch": 1.0301369863013699,
+      "grad_norm": 2.26204776763916,
+      "learning_rate": 3.6758732737611702e-06,
+      "loss": 1.8514,
+      "step": 940
+    },
+    {
+      "epoch": 1.0301369863013699,
+      "eval_loss": 2.0666027069091797,
+      "eval_runtime": 36.8336,
+      "eval_samples_per_second": 27.149,
+      "eval_steps_per_second": 1.14,
+      "step": 940
+    },
+    {
+      "epoch": 1.0312328767123287,
+      "grad_norm": 2.22210955619812,
+      "learning_rate": 3.673842404549147e-06,
+      "loss": 1.8743,
+      "step": 941
+    },
+    {
+      "epoch": 1.0323287671232877,
+      "grad_norm": 2.2371468544006348,
+      "learning_rate": 3.671811535337125e-06,
+      "loss": 1.8573,
+      "step": 942
+    },
+    {
+      "epoch": 1.0334246575342465,
+      "grad_norm": 2.346745014190674,
+      "learning_rate": 3.6697806661251016e-06,
+      "loss": 1.8614,
+      "step": 943
+    },
+    {
+      "epoch": 1.0345205479452055,
+      "grad_norm": 2.4765074253082275,
+      "learning_rate": 3.6677497969130793e-06,
+      "loss": 1.8654,
+      "step": 944
+    },
+    {
+      "epoch": 1.0356164383561643,
+      "grad_norm": 2.3023204803466797,
+      "learning_rate": 3.665718927701056e-06,
+      "loss": 1.9901,
+      "step": 945
+    },
+    {
+      "epoch": 1.0367123287671234,
+      "grad_norm": 2.322421073913574,
+      "learning_rate": 3.663688058489034e-06,
+      "loss": 1.8775,
+      "step": 946
+    },
+    {
+      "epoch": 1.0378082191780822,
+      "grad_norm": 2.2324275970458984,
+      "learning_rate": 3.6616571892770107e-06,
+      "loss": 1.9303,
+      "step": 947
+    },
+    {
+      "epoch": 1.038904109589041,
+      "grad_norm": 2.372067928314209,
+      "learning_rate": 3.659626320064988e-06,
+      "loss": 1.8673,
+      "step": 948
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 2.2404983043670654,
+      "learning_rate": 3.6575954508529653e-06,
+      "loss": 1.8025,
+      "step": 949
+    },
+    {
+      "epoch": 1.0410958904109588,
+      "grad_norm": 2.180673599243164,
+      "learning_rate": 3.6555645816409426e-06,
+      "loss": 1.8563,
+      "step": 950
+    },
+    {
+      "epoch": 1.0410958904109588,
+      "eval_loss": 2.061262845993042,
+      "eval_runtime": 36.7727,
+      "eval_samples_per_second": 27.194,
+      "eval_steps_per_second": 1.142,
+      "step": 950
+    },
+    {
+      "epoch": 1.0421917808219179,
+      "grad_norm": 2.299043893814087,
+      "learning_rate": 3.65353371242892e-06,
+      "loss": 1.8453,
+      "step": 951
+    },
+    {
+      "epoch": 1.0432876712328767,
+      "grad_norm": 2.4878089427948,
+      "learning_rate": 3.651502843216897e-06,
+      "loss": 1.8901,
+      "step": 952
+    },
+    {
+      "epoch": 1.0443835616438357,
+      "grad_norm": 2.150404930114746,
+      "learning_rate": 3.6494719740048744e-06,
+      "loss": 1.7371,
+      "step": 953
+    },
+    {
+      "epoch": 1.0454794520547945,
+      "grad_norm": 2.177750587463379,
+      "learning_rate": 3.6474411047928517e-06,
+      "loss": 1.8814,
+      "step": 954
+    },
+    {
+      "epoch": 1.0465753424657533,
+      "grad_norm": 2.230797052383423,
+      "learning_rate": 3.645410235580829e-06,
+      "loss": 1.8323,
+      "step": 955
+    },
+    {
+      "epoch": 1.0476712328767124,
+      "grad_norm": 2.2484378814697266,
+      "learning_rate": 3.6433793663688062e-06,
+      "loss": 1.8749,
+      "step": 956
+    },
+    {
+      "epoch": 1.0487671232876712,
+      "grad_norm": 2.201279640197754,
+      "learning_rate": 3.6413484971567835e-06,
+      "loss": 1.8518,
+      "step": 957
+    },
+    {
+      "epoch": 1.0498630136986302,
+      "grad_norm": 2.1125736236572266,
+      "learning_rate": 3.6393176279447608e-06,
+      "loss": 1.846,
+      "step": 958
+    },
+    {
+      "epoch": 1.050958904109589,
+      "grad_norm": 2.3078789710998535,
+      "learning_rate": 3.637286758732738e-06,
+      "loss": 1.8791,
+      "step": 959
+    },
+    {
+      "epoch": 1.0520547945205478,
+      "grad_norm": 2.2911148071289062,
+      "learning_rate": 3.6352558895207153e-06,
+      "loss": 1.8901,
+      "step": 960
+    },
+    {
+      "epoch": 1.0520547945205478,
+      "eval_loss": 2.0583066940307617,
+      "eval_runtime": 36.7535,
+      "eval_samples_per_second": 27.208,
+      "eval_steps_per_second": 1.143,
+      "step": 960
+    },
+    {
+      "epoch": 1.0531506849315069,
+      "grad_norm": 2.3990273475646973,
+      "learning_rate": 3.633225020308692e-06,
+      "loss": 1.9071,
+      "step": 961
+    },
+    {
+      "epoch": 1.0542465753424657,
+      "grad_norm": 2.358685255050659,
+      "learning_rate": 3.63119415109667e-06,
+      "loss": 1.8982,
+      "step": 962
+    },
+    {
+      "epoch": 1.0553424657534247,
+      "grad_norm": 2.1145992279052734,
+      "learning_rate": 3.6291632818846467e-06,
+      "loss": 1.8649,
+      "step": 963
+    },
+    {
+      "epoch": 1.0564383561643835,
+      "grad_norm": 2.2302935123443604,
+      "learning_rate": 3.6271324126726244e-06,
+      "loss": 1.8911,
+      "step": 964
+    },
+    {
+      "epoch": 1.0575342465753426,
+      "grad_norm": 2.344797372817993,
+      "learning_rate": 3.6251015434606013e-06,
+      "loss": 1.8,
+      "step": 965
+    },
+    {
+      "epoch": 1.0586301369863014,
+      "grad_norm": 2.2514073848724365,
+      "learning_rate": 3.623070674248579e-06,
+      "loss": 1.8128,
+      "step": 966
+    },
+    {
+      "epoch": 1.0597260273972602,
+      "grad_norm": 2.541221857070923,
+      "learning_rate": 3.621039805036556e-06,
+      "loss": 1.8504,
+      "step": 967
+    },
+    {
+      "epoch": 1.0608219178082192,
+      "grad_norm": 2.3211770057678223,
+      "learning_rate": 3.6190089358245335e-06,
+      "loss": 1.8329,
+      "step": 968
+    },
+    {
+      "epoch": 1.061917808219178,
+      "grad_norm": 2.1008756160736084,
+      "learning_rate": 3.6169780666125104e-06,
+      "loss": 1.8972,
+      "step": 969
+    },
+    {
+      "epoch": 1.063013698630137,
+      "grad_norm": 2.4148898124694824,
+      "learning_rate": 3.614947197400488e-06,
+      "loss": 1.8748,
+      "step": 970
+    },
+    {
+      "epoch": 1.063013698630137,
+      "eval_loss": 2.0592916011810303,
+      "eval_runtime": 36.7663,
+      "eval_samples_per_second": 27.199,
+      "eval_steps_per_second": 1.142,
+      "step": 970
+    },
+    {
+      "epoch": 1.0641095890410959,
+      "grad_norm": 2.4116220474243164,
+      "learning_rate": 3.612916328188465e-06,
+      "loss": 1.8915,
+      "step": 971
+    },
+    {
+      "epoch": 1.0652054794520547,
+      "grad_norm": 2.257411241531372,
+      "learning_rate": 3.610885458976442e-06,
+      "loss": 1.7901,
+      "step": 972
+    },
+    {
+      "epoch": 1.0663013698630137,
+      "grad_norm": 2.166875123977661,
+      "learning_rate": 3.6088545897644195e-06,
+      "loss": 1.8969,
+      "step": 973
+    },
+    {
+      "epoch": 1.0673972602739725,
+      "grad_norm": 2.464716911315918,
+      "learning_rate": 3.6068237205523963e-06,
+      "loss": 1.8844,
+      "step": 974
+    },
+    {
+      "epoch": 1.0684931506849316,
+      "grad_norm": 2.337334156036377,
+      "learning_rate": 3.604792851340374e-06,
+      "loss": 1.8192,
+      "step": 975
+    },
+    {
+      "epoch": 1.0695890410958904,
+      "grad_norm": 2.117663621902466,
+      "learning_rate": 3.602761982128351e-06,
+      "loss": 1.8233,
+      "step": 976
+    },
+    {
+      "epoch": 1.0706849315068494,
+      "grad_norm": 2.3353586196899414,
+      "learning_rate": 3.6007311129163286e-06,
+      "loss": 1.9133,
+      "step": 977
+    },
+    {
+      "epoch": 1.0717808219178082,
+      "grad_norm": 2.444279909133911,
+      "learning_rate": 3.5987002437043054e-06,
+      "loss": 1.8803,
+      "step": 978
+    },
+    {
+      "epoch": 1.072876712328767,
+      "grad_norm": 2.3458383083343506,
+      "learning_rate": 3.596669374492283e-06,
+      "loss": 1.8108,
+      "step": 979
+    },
+    {
+      "epoch": 1.073972602739726,
+      "grad_norm": 2.207977771759033,
+      "learning_rate": 3.59463850528026e-06,
+      "loss": 1.8423,
+      "step": 980
+    },
+    {
+      "epoch": 1.073972602739726,
+      "eval_loss": 2.0578773021698,
+      "eval_runtime": 36.8123,
+      "eval_samples_per_second": 27.165,
+      "eval_steps_per_second": 1.141,
+      "step": 980
+    },
+    {
+      "epoch": 1.0750684931506849,
+      "grad_norm": 2.378127336502075,
+      "learning_rate": 3.5926076360682377e-06,
+      "loss": 1.9268,
+      "step": 981
+    },
+    {
+      "epoch": 1.076164383561644,
+      "grad_norm": 2.248105525970459,
+      "learning_rate": 3.5905767668562145e-06,
+      "loss": 1.8111,
+      "step": 982
+    },
+    {
+      "epoch": 1.0772602739726027,
+      "grad_norm": 2.275592088699341,
+      "learning_rate": 3.5885458976441922e-06,
+      "loss": 1.9099,
+      "step": 983
+    },
+    {
+      "epoch": 1.0783561643835617,
+      "grad_norm": 3.108377456665039,
+      "learning_rate": 3.586515028432169e-06,
+      "loss": 1.8728,
+      "step": 984
+    },
+    {
+      "epoch": 1.0794520547945206,
+      "grad_norm": 2.2702999114990234,
+      "learning_rate": 3.5844841592201464e-06,
+      "loss": 1.9007,
+      "step": 985
+    },
+    {
+      "epoch": 1.0805479452054794,
+      "grad_norm": 3.218647003173828,
+      "learning_rate": 3.5824532900081237e-06,
+      "loss": 1.9157,
+      "step": 986
+    },
+    {
+      "epoch": 1.0816438356164384,
+      "grad_norm": 2.3393027782440186,
+      "learning_rate": 3.580422420796101e-06,
+      "loss": 1.7936,
+      "step": 987
+    },
+    {
+      "epoch": 1.0827397260273972,
+      "grad_norm": 2.420541286468506,
+      "learning_rate": 3.578391551584078e-06,
+      "loss": 1.8608,
+      "step": 988
+    },
+    {
+      "epoch": 1.0838356164383562,
+      "grad_norm": 2.3354556560516357,
+      "learning_rate": 3.5763606823720555e-06,
+      "loss": 1.8684,
+      "step": 989
+    },
+    {
+      "epoch": 1.084931506849315,
+      "grad_norm": 2.200282573699951,
+      "learning_rate": 3.5743298131600328e-06,
+      "loss": 1.7814,
+      "step": 990
+    },
+    {
+      "epoch": 1.084931506849315,
+      "eval_loss": 2.0565249919891357,
+      "eval_runtime": 36.952,
+      "eval_samples_per_second": 27.062,
+      "eval_steps_per_second": 1.137,
+      "step": 990
+    },
+    {
+      "epoch": 1.086027397260274,
+      "grad_norm": 2.149033308029175,
+      "learning_rate": 3.57229894394801e-06,
+      "loss": 1.8708,
+      "step": 991
+    },
+    {
+      "epoch": 1.087123287671233,
+      "grad_norm": 2.4663679599761963,
+      "learning_rate": 3.5702680747359873e-06,
+      "loss": 1.8947,
+      "step": 992
+    },
+    {
+      "epoch": 1.0882191780821917,
+      "grad_norm": 2.3142261505126953,
+      "learning_rate": 3.5682372055239646e-06,
+      "loss": 1.8727,
+      "step": 993
+    },
+    {
+      "epoch": 1.0893150684931507,
+      "grad_norm": 2.2223033905029297,
+      "learning_rate": 3.566206336311942e-06,
+      "loss": 1.8203,
+      "step": 994
+    },
+    {
+      "epoch": 1.0904109589041096,
+      "grad_norm": 2.423855781555176,
+      "learning_rate": 3.564175467099919e-06,
+      "loss": 1.8687,
+      "step": 995
+    },
+    {
+      "epoch": 1.0915068493150686,
+      "grad_norm": 2.4022302627563477,
+      "learning_rate": 3.5621445978878964e-06,
+      "loss": 1.8215,
+      "step": 996
+    },
+    {
+      "epoch": 1.0926027397260274,
+      "grad_norm": 2.378236770629883,
+      "learning_rate": 3.5601137286758737e-06,
+      "loss": 1.7994,
+      "step": 997
+    },
+    {
+      "epoch": 1.0936986301369862,
+      "grad_norm": 2.4499096870422363,
+      "learning_rate": 3.5580828594638505e-06,
+      "loss": 1.8779,
+      "step": 998
+    },
+    {
+      "epoch": 1.0947945205479452,
+      "grad_norm": 2.3181755542755127,
+      "learning_rate": 3.5560519902518282e-06,
+      "loss": 1.8341,
+      "step": 999
+    },
+    {
+      "epoch": 1.095890410958904,
+      "grad_norm": 2.2406623363494873,
+      "learning_rate": 3.554021121039805e-06,
+      "loss": 1.874,
+      "step": 1000
+    },
+    {
+      "epoch": 1.095890410958904,
+      "eval_loss": 2.0523672103881836,
+      "eval_runtime": 37.0454,
+      "eval_samples_per_second": 26.994,
+      "eval_steps_per_second": 1.134,
+      "step": 1000
+    },
+    {
+      "epoch": 1.096986301369863,
+      "grad_norm": 2.254667043685913,
+      "learning_rate": 3.551990251827783e-06,
+      "loss": 1.9017,
+      "step": 1001
+    },
+    {
+      "epoch": 1.098082191780822,
+      "grad_norm": 2.2846896648406982,
+      "learning_rate": 3.5499593826157596e-06,
+      "loss": 1.8505,
+      "step": 1002
+    },
+    {
+      "epoch": 1.099178082191781,
+      "grad_norm": 2.183469772338867,
+      "learning_rate": 3.5479285134037373e-06,
+      "loss": 1.8266,
+      "step": 1003
+    },
+    {
+      "epoch": 1.1002739726027397,
+      "grad_norm": 2.151798963546753,
+      "learning_rate": 3.545897644191714e-06,
+      "loss": 1.8487,
+      "step": 1004
+    },
+    {
+      "epoch": 1.1013698630136985,
+      "grad_norm": 2.2039146423339844,
+      "learning_rate": 3.543866774979692e-06,
+      "loss": 1.8327,
+      "step": 1005
+    },
+    {
+      "epoch": 1.1024657534246576,
+      "grad_norm": 2.2091665267944336,
+      "learning_rate": 3.5418359057676687e-06,
+      "loss": 1.8214,
+      "step": 1006
+    },
+    {
+      "epoch": 1.1035616438356164,
+      "grad_norm": 2.2821404933929443,
+      "learning_rate": 3.5398050365556464e-06,
+      "loss": 1.8591,
+      "step": 1007
+    },
+    {
+      "epoch": 1.1046575342465754,
+      "grad_norm": 2.1726973056793213,
+      "learning_rate": 3.5377741673436233e-06,
+      "loss": 1.8265,
+      "step": 1008
+    },
+    {
+      "epoch": 1.1057534246575342,
+      "grad_norm": 2.2870233058929443,
+      "learning_rate": 3.535743298131601e-06,
+      "loss": 1.8783,
+      "step": 1009
+    },
+    {
+      "epoch": 1.106849315068493,
+      "grad_norm": 2.148080587387085,
+      "learning_rate": 3.533712428919578e-06,
+      "loss": 1.8272,
+      "step": 1010
+    },
+    {
+      "epoch": 1.106849315068493,
+      "eval_loss": 2.049504041671753,
+      "eval_runtime": 36.8717,
+      "eval_samples_per_second": 27.121,
+      "eval_steps_per_second": 1.139,
+      "step": 1010
+    },
+    {
+      "epoch": 1.107945205479452,
+      "grad_norm": 2.182035446166992,
+      "learning_rate": 3.5316815597075547e-06,
+      "loss": 1.8771,
+      "step": 1011
+    },
+    {
+      "epoch": 1.1090410958904109,
+      "grad_norm": 2.135085344314575,
+      "learning_rate": 3.5296506904955324e-06,
+      "loss": 1.9031,
+      "step": 1012
+    },
+    {
+      "epoch": 1.11013698630137,
+      "grad_norm": 2.1017255783081055,
+      "learning_rate": 3.5276198212835093e-06,
+      "loss": 1.8341,
+      "step": 1013
+    },
+    {
+      "epoch": 1.1112328767123287,
+      "grad_norm": 2.2773115634918213,
+      "learning_rate": 3.525588952071487e-06,
+      "loss": 1.8564,
+      "step": 1014
+    },
+    {
+      "epoch": 1.1123287671232878,
+      "grad_norm": 2.2773115634918213,
+      "learning_rate": 3.525588952071487e-06,
+      "loss": 1.8971,
+      "step": 1015
+    },
+    {
+      "epoch": 1.1134246575342466,
+      "grad_norm": 2.273375988006592,
+      "learning_rate": 3.523558082859464e-06,
+      "loss": 1.9074,
+      "step": 1016
+    },
+    {
+      "epoch": 1.1145205479452054,
+      "grad_norm": 2.2055182456970215,
+      "learning_rate": 3.5215272136474415e-06,
+      "loss": 1.8724,
+      "step": 1017
+    },
+    {
+      "epoch": 1.1156164383561644,
+      "grad_norm": 2.246126651763916,
+      "learning_rate": 3.5194963444354184e-06,
+      "loss": 1.8387,
+      "step": 1018
+    },
+    {
+      "epoch": 1.1167123287671232,
+      "grad_norm": 2.3076183795928955,
+      "learning_rate": 3.517465475223396e-06,
+      "loss": 1.8387,
+      "step": 1019
+    },
+    {
+      "epoch": 1.1178082191780823,
+      "grad_norm": 2.3365488052368164,
+      "learning_rate": 3.515434606011373e-06,
+      "loss": 1.8063,
+      "step": 1020
+    },
+    {
+      "epoch": 1.1178082191780823,
+      "eval_loss": 2.0528781414031982,
+      "eval_runtime": 36.8885,
+      "eval_samples_per_second": 27.109,
+      "eval_steps_per_second": 1.139,
+      "step": 1020
+    },
+    {
+      "epoch": 1.118904109589041,
+      "grad_norm": 2.319324493408203,
+      "learning_rate": 3.5134037367993506e-06,
+      "loss": 1.8843,
+      "step": 1021
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 2.238720655441284,
+      "learning_rate": 3.5113728675873275e-06,
+      "loss": 1.9375,
+      "step": 1022
+    },
+    {
+      "epoch": 1.121095890410959,
+      "grad_norm": 2.2824833393096924,
+      "learning_rate": 3.5093419983753047e-06,
+      "loss": 1.8749,
+      "step": 1023
+    },
+    {
+      "epoch": 1.1221917808219177,
+      "grad_norm": 2.5120770931243896,
+      "learning_rate": 3.507311129163282e-06,
+      "loss": 1.8774,
+      "step": 1024
+    },
+    {
+      "epoch": 1.1232876712328768,
+      "grad_norm": 2.1762094497680664,
+      "learning_rate": 3.5052802599512593e-06,
+      "loss": 1.8018,
+      "step": 1025
+    },
+    {
+      "epoch": 1.1243835616438356,
+      "grad_norm": 2.2530343532562256,
+      "learning_rate": 3.5032493907392366e-06,
+      "loss": 1.8451,
+      "step": 1026
+    },
+    {
+      "epoch": 1.1254794520547946,
+      "grad_norm": 2.21649169921875,
+      "learning_rate": 3.501218521527214e-06,
+      "loss": 1.8613,
+      "step": 1027
+    },
+    {
+      "epoch": 1.1265753424657534,
+      "grad_norm": 2.306431293487549,
+      "learning_rate": 3.499187652315191e-06,
+      "loss": 1.7947,
+      "step": 1028
+    },
+    {
+      "epoch": 1.1276712328767124,
+      "grad_norm": 2.203005313873291,
+      "learning_rate": 3.4971567831031684e-06,
+      "loss": 1.8483,
+      "step": 1029
+    },
+    {
+      "epoch": 1.1287671232876713,
+      "grad_norm": 2.1645009517669678,
+      "learning_rate": 3.4951259138911457e-06,
+      "loss": 1.8915,
+      "step": 1030
+    },
+    {
+      "epoch": 1.1287671232876713,
+      "eval_loss": 2.053393602371216,
+      "eval_runtime": 36.855,
+      "eval_samples_per_second": 27.133,
+      "eval_steps_per_second": 1.14,
+      "step": 1030
+    },
+    {
+      "epoch": 1.12986301369863,
+      "grad_norm": 2.2863385677337646,
+      "learning_rate": 3.493095044679123e-06,
+      "loss": 1.8073,
+      "step": 1031
+    },
+    {
+      "epoch": 1.130958904109589,
+      "grad_norm": 2.1439812183380127,
+      "learning_rate": 3.4910641754671002e-06,
+      "loss": 1.9191,
+      "step": 1032
+    },
+    {
+      "epoch": 1.132054794520548,
+      "grad_norm": 2.224228858947754,
+      "learning_rate": 3.4890333062550775e-06,
+      "loss": 1.7478,
+      "step": 1033
+    },
+    {
+      "epoch": 1.133150684931507,
+      "grad_norm": 2.319059133529663,
+      "learning_rate": 3.4870024370430548e-06,
+      "loss": 1.8779,
+      "step": 1034
+    },
+    {
+      "epoch": 1.1342465753424658,
+      "grad_norm": 2.5003139972686768,
+      "learning_rate": 3.484971567831032e-06,
+      "loss": 1.776,
+      "step": 1035
+    },
+    {
+      "epoch": 1.1353424657534246,
+      "grad_norm": 2.2032580375671387,
+      "learning_rate": 3.482940698619009e-06,
+      "loss": 1.8221,
+      "step": 1036
+    },
+    {
+      "epoch": 1.1364383561643836,
+      "grad_norm": 2.5253560543060303,
+      "learning_rate": 3.4809098294069866e-06,
+      "loss": 1.8952,
+      "step": 1037
+    },
+    {
+      "epoch": 1.1375342465753424,
+      "grad_norm": 2.3215854167938232,
+      "learning_rate": 3.4788789601949634e-06,
+      "loss": 1.847,
+      "step": 1038
+    },
+    {
+      "epoch": 1.1386301369863014,
+      "grad_norm": 2.2543280124664307,
+      "learning_rate": 3.476848090982941e-06,
+      "loss": 1.8689,
+      "step": 1039
+    },
+    {
+      "epoch": 1.1397260273972603,
+      "grad_norm": 2.1592042446136475,
+      "learning_rate": 3.474817221770918e-06,
+      "loss": 1.8375,
+      "step": 1040
+    },
+    {
+      "epoch": 1.1397260273972603,
+      "eval_loss": 2.0524044036865234,
+      "eval_runtime": 36.8163,
+      "eval_samples_per_second": 27.162,
+      "eval_steps_per_second": 1.141,
+      "step": 1040
+    },
+    {
+      "epoch": 1.140821917808219,
+      "grad_norm": 2.2874810695648193,
+      "learning_rate": 3.4727863525588957e-06,
+      "loss": 1.9442,
+      "step": 1041
+    },
+    {
+      "epoch": 1.141917808219178,
+      "grad_norm": 2.288395404815674,
+      "learning_rate": 3.4707554833468725e-06,
+      "loss": 1.8881,
+      "step": 1042
+    },
+    {
+      "epoch": 1.143013698630137,
+      "grad_norm": 2.2264153957366943,
+      "learning_rate": 3.4687246141348502e-06,
+      "loss": 1.8518,
+      "step": 1043
+    },
+    {
+      "epoch": 1.144109589041096,
+      "grad_norm": 2.4413652420043945,
+      "learning_rate": 3.466693744922827e-06,
+      "loss": 1.8146,
+      "step": 1044
+    },
+    {
+      "epoch": 1.1452054794520548,
+      "grad_norm": 2.3597898483276367,
+      "learning_rate": 3.464662875710805e-06,
+      "loss": 1.8421,
+      "step": 1045
+    },
+    {
+      "epoch": 1.1463013698630138,
+      "grad_norm": 2.2489402294158936,
+      "learning_rate": 3.4626320064987817e-06,
+      "loss": 1.8643,
+      "step": 1046
+    },
+    {
+      "epoch": 1.1473972602739726,
+      "grad_norm": 2.3958940505981445,
+      "learning_rate": 3.4606011372867594e-06,
+      "loss": 1.8522,
+      "step": 1047
+    },
+    {
+      "epoch": 1.1484931506849314,
+      "grad_norm": 2.295349597930908,
+      "learning_rate": 3.458570268074736e-06,
+      "loss": 1.8677,
+      "step": 1048
+    },
+    {
+      "epoch": 1.1495890410958904,
+      "grad_norm": 2.2571280002593994,
+      "learning_rate": 3.4565393988627135e-06,
+      "loss": 1.8393,
+      "step": 1049
+    },
+    {
+      "epoch": 1.1506849315068493,
+      "grad_norm": 2.173983335494995,
+      "learning_rate": 3.4545085296506908e-06,
+      "loss": 1.8267,
+      "step": 1050
+    },
+    {
+      "epoch": 1.1506849315068493,
+      "eval_loss": 2.0489604473114014,
+      "eval_runtime": 36.7678,
+      "eval_samples_per_second": 27.198,
+      "eval_steps_per_second": 1.142,
+      "step": 1050
+    },
+    {
+      "epoch": 1.1517808219178083,
+      "grad_norm": 2.3395209312438965,
+      "learning_rate": 3.452477660438668e-06,
+      "loss": 1.8032,
+      "step": 1051
+    },
+    {
+      "epoch": 1.152876712328767,
+      "grad_norm": 2.1894757747650146,
+      "learning_rate": 3.4504467912266453e-06,
+      "loss": 1.8834,
+      "step": 1052
+    },
+    {
+      "epoch": 1.1539726027397261,
+      "grad_norm": 2.1978437900543213,
+      "learning_rate": 3.4484159220146226e-06,
+      "loss": 1.7897,
+      "step": 1053
+    },
+    {
+      "epoch": 1.155068493150685,
+      "grad_norm": 2.275792360305786,
+      "learning_rate": 3.4463850528026e-06,
+      "loss": 1.8566,
+      "step": 1054
+    },
+    {
+      "epoch": 1.1561643835616437,
+      "grad_norm": 2.2659800052642822,
+      "learning_rate": 3.444354183590577e-06,
+      "loss": 1.8307,
+      "step": 1055
+    },
+    {
+      "epoch": 1.1572602739726028,
+      "grad_norm": 2.232149600982666,
+      "learning_rate": 3.4423233143785544e-06,
+      "loss": 1.8701,
+      "step": 1056
+    },
+    {
+      "epoch": 1.1583561643835616,
+      "grad_norm": 2.3636281490325928,
+      "learning_rate": 3.4402924451665317e-06,
+      "loss": 1.8008,
+      "step": 1057
+    },
+    {
+      "epoch": 1.1594520547945206,
+      "grad_norm": 2.271533489227295,
+      "learning_rate": 3.438261575954509e-06,
+      "loss": 1.7751,
+      "step": 1058
+    },
+    {
+      "epoch": 1.1605479452054794,
+      "grad_norm": 2.3286566734313965,
+      "learning_rate": 3.4362307067424862e-06,
+      "loss": 1.865,
+      "step": 1059
+    },
+    {
+      "epoch": 1.1616438356164385,
+      "grad_norm": 2.46665096282959,
+      "learning_rate": 3.434199837530463e-06,
+      "loss": 1.9001,
+      "step": 1060
+    },
+    {
+      "epoch": 1.1616438356164385,
+      "eval_loss": 2.0477516651153564,
+      "eval_runtime": 36.7818,
+      "eval_samples_per_second": 27.187,
+      "eval_steps_per_second": 1.142,
+      "step": 1060
+    },
+    {
+      "epoch": 1.1627397260273973,
+      "grad_norm": 2.307744264602661,
+      "learning_rate": 3.432168968318441e-06,
+      "loss": 1.8637,
+      "step": 1061
+    },
+    {
+      "epoch": 1.163835616438356,
+      "grad_norm": 2.3935985565185547,
+      "learning_rate": 3.4301380991064176e-06,
+      "loss": 1.7867,
+      "step": 1062
+    },
+    {
+      "epoch": 1.1649315068493151,
+      "grad_norm": 2.290971040725708,
+      "learning_rate": 3.4281072298943953e-06,
+      "loss": 1.8708,
+      "step": 1063
+    },
+    {
+      "epoch": 1.166027397260274,
+      "grad_norm": 2.191629409790039,
+      "learning_rate": 3.426076360682372e-06,
+      "loss": 1.9031,
+      "step": 1064
+    },
+    {
+      "epoch": 1.167123287671233,
+      "grad_norm": 2.284426689147949,
+      "learning_rate": 3.42404549147035e-06,
+      "loss": 1.9026,
+      "step": 1065
+    },
+    {
+      "epoch": 1.1682191780821918,
+      "grad_norm": 2.2567849159240723,
+      "learning_rate": 3.4220146222583267e-06,
+      "loss": 1.8507,
+      "step": 1066
+    },
+    {
+      "epoch": 1.1693150684931506,
+      "grad_norm": 2.2093160152435303,
+      "learning_rate": 3.4199837530463044e-06,
+      "loss": 1.8001,
+      "step": 1067
+    },
+    {
+      "epoch": 1.1704109589041096,
+      "grad_norm": 2.092560291290283,
+      "learning_rate": 3.4179528838342813e-06,
+      "loss": 1.8393,
+      "step": 1068
+    },
+    {
+      "epoch": 1.1715068493150684,
+      "grad_norm": 2.313749074935913,
+      "learning_rate": 3.415922014622259e-06,
+      "loss": 1.9385,
+      "step": 1069
+    },
+    {
+      "epoch": 1.1726027397260275,
+      "grad_norm": 2.488673210144043,
+      "learning_rate": 3.413891145410236e-06,
+      "loss": 1.8924,
+      "step": 1070
+    },
+    {
+      "epoch": 1.1726027397260275,
+      "eval_loss": 2.0482401847839355,
+      "eval_runtime": 36.7919,
+      "eval_samples_per_second": 27.18,
+      "eval_steps_per_second": 1.142,
+      "step": 1070
+    },
+    {
+      "epoch": 1.1736986301369863,
+      "grad_norm": 2.0804412364959717,
+      "learning_rate": 3.4118602761982135e-06,
+      "loss": 1.881,
+      "step": 1071
+    },
+    {
+      "epoch": 1.174794520547945,
+      "grad_norm": 2.165426254272461,
+      "learning_rate": 3.4098294069861904e-06,
+      "loss": 1.9035,
+      "step": 1072
+    },
+    {
+      "epoch": 1.1758904109589041,
+      "grad_norm": 2.3034675121307373,
+      "learning_rate": 3.4077985377741673e-06,
+      "loss": 1.8554,
+      "step": 1073
+    },
+    {
+      "epoch": 1.176986301369863,
+      "grad_norm": 2.303159713745117,
+      "learning_rate": 3.405767668562145e-06,
+      "loss": 1.8255,
+      "step": 1074
+    },
+    {
+      "epoch": 1.178082191780822,
+      "grad_norm": 2.2402753829956055,
+      "learning_rate": 3.403736799350122e-06,
+      "loss": 1.9087,
+      "step": 1075
+    },
+    {
+      "epoch": 1.1791780821917808,
+      "grad_norm": 2.251638650894165,
+      "learning_rate": 3.4017059301380995e-06,
+      "loss": 1.8878,
+      "step": 1076
+    },
+    {
+      "epoch": 1.1802739726027398,
+      "grad_norm": 2.2661657333374023,
+      "learning_rate": 3.3996750609260764e-06,
+      "loss": 1.8244,
+      "step": 1077
+    },
+    {
+      "epoch": 1.1813698630136986,
+      "grad_norm": 2.3280904293060303,
+      "learning_rate": 3.397644191714054e-06,
+      "loss": 1.9328,
+      "step": 1078
+    },
+    {
+      "epoch": 1.1824657534246574,
+      "grad_norm": 2.315779447555542,
+      "learning_rate": 3.395613322502031e-06,
+      "loss": 1.8289,
+      "step": 1079
+    },
+    {
+      "epoch": 1.1835616438356165,
+      "grad_norm": 2.136967182159424,
+      "learning_rate": 3.3935824532900086e-06,
+      "loss": 1.8307,
+      "step": 1080
+    },
+    {
+      "epoch": 1.1835616438356165,
+      "eval_loss": 2.0473902225494385,
+      "eval_runtime": 36.8094,
+      "eval_samples_per_second": 27.167,
+      "eval_steps_per_second": 1.141,
+      "step": 1080
+    },
+    {
+      "epoch": 1.1846575342465753,
+      "grad_norm": 2.3701720237731934,
+      "learning_rate": 3.3915515840779855e-06,
+      "loss": 1.8123,
+      "step": 1081
+    },
+    {
+      "epoch": 1.1857534246575343,
+      "grad_norm": 2.2662320137023926,
+      "learning_rate": 3.389520714865963e-06,
+      "loss": 1.85,
+      "step": 1082
+    },
+    {
+      "epoch": 1.1868493150684931,
+      "grad_norm": 2.2151782512664795,
+      "learning_rate": 3.38748984565394e-06,
+      "loss": 1.845,
+      "step": 1083
+    },
+    {
+      "epoch": 1.1879452054794521,
+      "grad_norm": 2.2209115028381348,
+      "learning_rate": 3.3854589764419177e-06,
+      "loss": 1.8787,
+      "step": 1084
+    },
+    {
+      "epoch": 1.189041095890411,
+      "grad_norm": 2.4760477542877197,
+      "learning_rate": 3.3834281072298946e-06,
+      "loss": 1.818,
+      "step": 1085
+    },
+    {
+      "epoch": 1.1901369863013698,
+      "grad_norm": 2.309034585952759,
+      "learning_rate": 3.381397238017872e-06,
+      "loss": 1.8387,
+      "step": 1086
+    },
+    {
+      "epoch": 1.1912328767123288,
+      "grad_norm": 2.2174596786499023,
+      "learning_rate": 3.379366368805849e-06,
+      "loss": 1.8135,
+      "step": 1087
+    },
+    {
+      "epoch": 1.1923287671232876,
+      "grad_norm": 2.3043556213378906,
+      "learning_rate": 3.3773354995938264e-06,
+      "loss": 1.9156,
+      "step": 1088
+    },
+    {
+      "epoch": 1.1934246575342466,
+      "grad_norm": 2.2240524291992188,
+      "learning_rate": 3.3753046303818037e-06,
+      "loss": 1.8636,
+      "step": 1089
+    },
+    {
+      "epoch": 1.1945205479452055,
+      "grad_norm": 2.1108691692352295,
+      "learning_rate": 3.373273761169781e-06,
+      "loss": 1.8458,
+      "step": 1090
+    },
+    {
+      "epoch": 1.1945205479452055,
+      "eval_loss": 2.04378604888916,
+      "eval_runtime": 36.9058,
+      "eval_samples_per_second": 27.096,
+      "eval_steps_per_second": 1.138,
+      "step": 1090
+    },
+    {
+      "epoch": 1.1956164383561645,
+      "grad_norm": 2.1893398761749268,
+      "learning_rate": 3.3712428919577582e-06,
+      "loss": 1.8446,
+      "step": 1091
+    },
+    {
+      "epoch": 1.1967123287671233,
+      "grad_norm": 2.305298089981079,
+      "learning_rate": 3.3692120227457355e-06,
+      "loss": 1.8283,
+      "step": 1092
+    },
+    {
+      "epoch": 1.1978082191780821,
+      "grad_norm": 2.364433765411377,
+      "learning_rate": 3.3671811535337128e-06,
+      "loss": 1.8708,
+      "step": 1093
+    },
+    {
+      "epoch": 1.1989041095890411,
+      "grad_norm": 2.259892225265503,
+      "learning_rate": 3.36515028432169e-06,
+      "loss": 1.7966,
+      "step": 1094
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 2.2487263679504395,
+      "learning_rate": 3.3631194151096673e-06,
+      "loss": 1.8238,
+      "step": 1095
+    },
+    {
+      "epoch": 1.201095890410959,
+      "grad_norm": 2.1592557430267334,
+      "learning_rate": 3.3610885458976446e-06,
+      "loss": 1.8651,
+      "step": 1096
+    },
+    {
+      "epoch": 1.2021917808219178,
+      "grad_norm": 2.257737159729004,
+      "learning_rate": 3.359057676685622e-06,
+      "loss": 1.8926,
+      "step": 1097
+    },
+    {
+      "epoch": 1.2032876712328768,
+      "grad_norm": 2.0702707767486572,
+      "learning_rate": 3.357026807473599e-06,
+      "loss": 1.8833,
+      "step": 1098
+    },
+    {
+      "epoch": 1.2043835616438356,
+      "grad_norm": 2.1568455696105957,
+      "learning_rate": 3.354995938261576e-06,
+      "loss": 1.7732,
+      "step": 1099
+    },
+    {
+      "epoch": 1.2054794520547945,
+      "grad_norm": 2.294715404510498,
+      "learning_rate": 3.3529650690495537e-06,
+      "loss": 1.8205,
+      "step": 1100
+    },
+    {
+      "epoch": 1.2054794520547945,
+      "eval_loss": 2.044201612472534,
+      "eval_runtime": 36.9861,
+      "eval_samples_per_second": 27.037,
+      "eval_steps_per_second": 1.136,
+      "step": 1100
+    },
+    {
+      "epoch": 1.2065753424657535,
+      "grad_norm": 2.3234570026397705,
+      "learning_rate": 3.3509341998375305e-06,
+      "loss": 1.7935,
+      "step": 1101
+    },
+    {
+      "epoch": 1.2076712328767123,
+      "grad_norm": 2.307224750518799,
+      "learning_rate": 3.3489033306255082e-06,
+      "loss": 1.8254,
+      "step": 1102
+    },
+    {
+      "epoch": 1.208767123287671,
+      "grad_norm": 2.3725457191467285,
+      "learning_rate": 3.346872461413485e-06,
+      "loss": 1.8493,
+      "step": 1103
+    },
+    {
+      "epoch": 1.2098630136986301,
+      "grad_norm": 2.2057435512542725,
+      "learning_rate": 3.344841592201463e-06,
+      "loss": 1.8241,
+      "step": 1104
+    },
+    {
+      "epoch": 1.210958904109589,
+      "grad_norm": 2.232947826385498,
+      "learning_rate": 3.3428107229894397e-06,
+      "loss": 1.8057,
+      "step": 1105
+    },
+    {
+      "epoch": 1.212054794520548,
+      "grad_norm": 2.4268925189971924,
+      "learning_rate": 3.3407798537774174e-06,
+      "loss": 1.8334,
+      "step": 1106
+    },
+    {
+      "epoch": 1.2131506849315068,
+      "grad_norm": 2.3134701251983643,
+      "learning_rate": 3.338748984565394e-06,
+      "loss": 1.8516,
+      "step": 1107
+    },
+    {
+      "epoch": 1.2142465753424658,
+      "grad_norm": 2.375873327255249,
+      "learning_rate": 3.336718115353372e-06,
+      "loss": 1.8942,
+      "step": 1108
+    },
+    {
+      "epoch": 1.2153424657534246,
+      "grad_norm": 2.622528076171875,
+      "learning_rate": 3.3346872461413488e-06,
+      "loss": 1.9027,
+      "step": 1109
+    },
+    {
+      "epoch": 1.2164383561643834,
+      "grad_norm": 2.290865898132324,
+      "learning_rate": 3.3326563769293256e-06,
+      "loss": 1.8258,
+      "step": 1110
+    },
+    {
+      "epoch": 1.2164383561643834,
+      "eval_loss": 2.0408973693847656,
+      "eval_runtime": 36.8596,
+      "eval_samples_per_second": 27.13,
+      "eval_steps_per_second": 1.139,
+      "step": 1110
+    },
+    {
+      "epoch": 1.2175342465753425,
+      "grad_norm": 2.1596908569335938,
+      "learning_rate": 3.3306255077173033e-06,
+      "loss": 1.8066,
+      "step": 1111
+    },
+    {
+      "epoch": 1.2186301369863013,
+      "grad_norm": 2.4652187824249268,
+      "learning_rate": 3.32859463850528e-06,
+      "loss": 1.8372,
+      "step": 1112
+    },
+    {
+      "epoch": 1.2197260273972603,
+      "grad_norm": 2.282440185546875,
+      "learning_rate": 3.326563769293258e-06,
+      "loss": 1.8535,
+      "step": 1113
+    },
+    {
+      "epoch": 1.2208219178082191,
+      "grad_norm": 2.2390308380126953,
+      "learning_rate": 3.3245329000812347e-06,
+      "loss": 1.7719,
+      "step": 1114
+    },
+    {
+      "epoch": 1.2219178082191782,
+      "grad_norm": 2.368565320968628,
+      "learning_rate": 3.3225020308692124e-06,
+      "loss": 1.8207,
+      "step": 1115
+    },
+    {
+      "epoch": 1.223013698630137,
+      "grad_norm": 2.463942766189575,
+      "learning_rate": 3.3204711616571893e-06,
+      "loss": 1.8402,
+      "step": 1116
+    },
+    {
+      "epoch": 1.2241095890410958,
+      "grad_norm": 2.266096830368042,
+      "learning_rate": 3.318440292445167e-06,
+      "loss": 1.7854,
+      "step": 1117
+    },
+    {
+      "epoch": 1.2252054794520548,
+      "grad_norm": 2.3463494777679443,
+      "learning_rate": 3.316409423233144e-06,
+      "loss": 1.8449,
+      "step": 1118
+    },
+    {
+      "epoch": 1.2263013698630136,
+      "grad_norm": 2.381727933883667,
+      "learning_rate": 3.3143785540211215e-06,
+      "loss": 1.8085,
+      "step": 1119
+    },
+    {
+      "epoch": 1.2273972602739727,
+      "grad_norm": 2.3251113891601562,
+      "learning_rate": 3.3123476848090984e-06,
+      "loss": 1.8922,
+      "step": 1120
+    },
+    {
+      "epoch": 1.2273972602739727,
+      "eval_loss": 2.0425429344177246,
+      "eval_runtime": 36.851,
+      "eval_samples_per_second": 27.136,
+      "eval_steps_per_second": 1.14,
+      "step": 1120
+    },
+    {
+      "epoch": 1.2284931506849315,
+      "grad_norm": 2.2314298152923584,
+      "learning_rate": 3.310316815597076e-06,
+      "loss": 1.8474,
+      "step": 1121
+    },
+    {
+      "epoch": 1.2295890410958905,
+      "grad_norm": 2.1847472190856934,
+      "learning_rate": 3.308285946385053e-06,
+      "loss": 1.8454,
+      "step": 1122
+    },
+    {
+      "epoch": 1.2306849315068493,
+      "grad_norm": 2.249422788619995,
+      "learning_rate": 3.30625507717303e-06,
+      "loss": 1.9257,
+      "step": 1123
+    },
+    {
+      "epoch": 1.2317808219178081,
+      "grad_norm": 2.223930835723877,
+      "learning_rate": 3.3042242079610075e-06,
+      "loss": 1.7816,
+      "step": 1124
+    },
+    {
+      "epoch": 1.2328767123287672,
+      "grad_norm": 2.3209826946258545,
+      "learning_rate": 3.3021933387489847e-06,
+      "loss": 1.8773,
+      "step": 1125
+    },
+    {
+      "epoch": 1.233972602739726,
+      "grad_norm": 2.2288055419921875,
+      "learning_rate": 3.300162469536962e-06,
+      "loss": 1.8931,
+      "step": 1126
+    },
+    {
+      "epoch": 1.235068493150685,
+      "grad_norm": 2.201449155807495,
+      "learning_rate": 3.2981316003249393e-06,
+      "loss": 1.8149,
+      "step": 1127
+    },
+    {
+      "epoch": 1.2361643835616438,
+      "grad_norm": 2.215824842453003,
+      "learning_rate": 3.2961007311129166e-06,
+      "loss": 1.8949,
+      "step": 1128
+    },
+    {
+      "epoch": 1.2372602739726029,
+      "grad_norm": 2.133671522140503,
+      "learning_rate": 3.294069861900894e-06,
+      "loss": 1.8669,
+      "step": 1129
+    },
+    {
+      "epoch": 1.2383561643835617,
+      "grad_norm": 2.2195792198181152,
+      "learning_rate": 3.292038992688871e-06,
+      "loss": 1.9245,
+      "step": 1130
+    },
+    {
+      "epoch": 1.2383561643835617,
+      "eval_loss": 2.0442771911621094,
+      "eval_runtime": 36.8077,
+      "eval_samples_per_second": 27.168,
+      "eval_steps_per_second": 1.141,
+      "step": 1130
+    },
+    {
+      "epoch": 1.2394520547945205,
+      "grad_norm": 2.310866117477417,
+      "learning_rate": 3.2900081234768484e-06,
+      "loss": 1.8523,
+      "step": 1131
+    },
+    {
+      "epoch": 1.2405479452054795,
+      "grad_norm": 2.318300247192383,
+      "learning_rate": 3.2879772542648257e-06,
+      "loss": 1.8665,
+      "step": 1132
+    },
+    {
+      "epoch": 1.2416438356164383,
+      "grad_norm": 2.082564115524292,
+      "learning_rate": 3.285946385052803e-06,
+      "loss": 1.8962,
+      "step": 1133
+    },
+    {
+      "epoch": 1.2427397260273974,
+      "grad_norm": 2.1982104778289795,
+      "learning_rate": 3.2839155158407802e-06,
+      "loss": 1.7722,
+      "step": 1134
+    },
+    {
+      "epoch": 1.2438356164383562,
+      "grad_norm": 2.330782890319824,
+      "learning_rate": 3.2818846466287575e-06,
+      "loss": 1.8787,
+      "step": 1135
+    },
+    {
+      "epoch": 1.244931506849315,
+      "grad_norm": 2.136747121810913,
+      "learning_rate": 3.2798537774167344e-06,
+      "loss": 1.7739,
+      "step": 1136
+    },
+    {
+      "epoch": 1.246027397260274,
+      "grad_norm": 2.205862283706665,
+      "learning_rate": 3.277822908204712e-06,
+      "loss": 1.9127,
+      "step": 1137
+    },
+    {
+      "epoch": 1.2471232876712328,
+      "grad_norm": 2.2060093879699707,
+      "learning_rate": 3.275792038992689e-06,
+      "loss": 1.8379,
+      "step": 1138
+    },
+    {
+      "epoch": 1.2482191780821918,
+      "grad_norm": 2.255772829055786,
+      "learning_rate": 3.2737611697806666e-06,
+      "loss": 1.7931,
+      "step": 1139
+    },
+    {
+      "epoch": 1.2493150684931507,
+      "grad_norm": 2.240095615386963,
+      "learning_rate": 3.2717303005686435e-06,
+      "loss": 1.8466,
+      "step": 1140
+    },
+    {
+      "epoch": 1.2493150684931507,
+      "eval_loss": 2.0429375171661377,
+      "eval_runtime": 36.7983,
+      "eval_samples_per_second": 27.175,
+      "eval_steps_per_second": 1.141,
+      "step": 1140
+    },
+    {
+      "epoch": 1.2504109589041095,
+      "grad_norm": 2.1491949558258057,
+      "learning_rate": 3.269699431356621e-06,
+      "loss": 1.8137,
+      "step": 1141
+    },
+    {
+      "epoch": 1.2515068493150685,
+      "grad_norm": 2.3854610919952393,
+      "learning_rate": 3.267668562144598e-06,
+      "loss": 1.8486,
+      "step": 1142
+    },
+    {
+      "epoch": 1.2526027397260273,
+      "grad_norm": 2.3498709201812744,
+      "learning_rate": 3.2656376929325757e-06,
+      "loss": 1.837,
+      "step": 1143
+    },
+    {
+      "epoch": 1.2536986301369863,
+      "grad_norm": 2.1651649475097656,
+      "learning_rate": 3.2636068237205526e-06,
+      "loss": 1.8,
+      "step": 1144
+    },
+    {
+      "epoch": 1.2547945205479452,
+      "grad_norm": 2.2500200271606445,
+      "learning_rate": 3.2615759545085303e-06,
+      "loss": 1.8691,
+      "step": 1145
+    },
+    {
+      "epoch": 1.2558904109589042,
+      "grad_norm": 2.2178192138671875,
+      "learning_rate": 3.259545085296507e-06,
+      "loss": 1.8345,
+      "step": 1146
+    },
+    {
+      "epoch": 1.256986301369863,
+      "grad_norm": 2.238114595413208,
+      "learning_rate": 3.257514216084484e-06,
+      "loss": 1.8935,
+      "step": 1147
+    },
+    {
+      "epoch": 1.2580821917808218,
+      "grad_norm": 2.37042498588562,
+      "learning_rate": 3.2554833468724617e-06,
+      "loss": 1.7872,
+      "step": 1148
+    },
+    {
+      "epoch": 1.2591780821917808,
+      "grad_norm": 2.1831908226013184,
+      "learning_rate": 3.2534524776604385e-06,
+      "loss": 1.8913,
+      "step": 1149
+    },
+    {
+      "epoch": 1.2602739726027397,
+      "grad_norm": 2.251382827758789,
+      "learning_rate": 3.2514216084484162e-06,
+      "loss": 1.8611,
+      "step": 1150
+    },
+    {
+      "epoch": 1.2602739726027397,
+      "eval_loss": 2.041306257247925,
+      "eval_runtime": 36.8199,
+      "eval_samples_per_second": 27.159,
+      "eval_steps_per_second": 1.141,
+      "step": 1150
+    },
+    {
+      "epoch": 1.2613698630136987,
+      "grad_norm": 2.2690420150756836,
+      "learning_rate": 3.249390739236393e-06,
+      "loss": 1.8283,
+      "step": 1151
+    },
+    {
+      "epoch": 1.2624657534246575,
+      "grad_norm": 2.255833387374878,
+      "learning_rate": 3.2473598700243708e-06,
+      "loss": 1.8563,
+      "step": 1152
+    },
+    {
+      "epoch": 1.2635616438356165,
+      "grad_norm": 2.1362855434417725,
+      "learning_rate": 3.2453290008123476e-06,
+      "loss": 1.809,
+      "step": 1153
+    },
+    {
+      "epoch": 1.2646575342465753,
+      "grad_norm": 2.1227219104766846,
+      "learning_rate": 3.2432981316003253e-06,
+      "loss": 1.848,
+      "step": 1154
+    },
+    {
+      "epoch": 1.2657534246575342,
+      "grad_norm": 2.3639719486236572,
+      "learning_rate": 3.241267262388302e-06,
+      "loss": 1.8472,
+      "step": 1155
+    },
+    {
+      "epoch": 1.2668493150684932,
+      "grad_norm": 2.334404468536377,
+      "learning_rate": 3.23923639317628e-06,
+      "loss": 1.8073,
+      "step": 1156
+    },
+    {
+      "epoch": 1.267945205479452,
+      "grad_norm": 2.231656312942505,
+      "learning_rate": 3.2372055239642567e-06,
+      "loss": 1.8446,
+      "step": 1157
+    },
+    {
+      "epoch": 1.269041095890411,
+      "grad_norm": 2.371375560760498,
+      "learning_rate": 3.2351746547522344e-06,
+      "loss": 1.8002,
+      "step": 1158
+    },
+    {
+      "epoch": 1.2701369863013698,
+      "grad_norm": 2.5212879180908203,
+      "learning_rate": 3.2331437855402113e-06,
+      "loss": 1.863,
+      "step": 1159
+    },
+    {
+      "epoch": 1.2712328767123289,
+      "grad_norm": 2.3285911083221436,
+      "learning_rate": 3.2311129163281885e-06,
+      "loss": 1.9101,
+      "step": 1160
+    },
+    {
+      "epoch": 1.2712328767123289,
+      "eval_loss": 2.038487195968628,
+      "eval_runtime": 36.8002,
+      "eval_samples_per_second": 27.174,
+      "eval_steps_per_second": 1.141,
+      "step": 1160
+    },
+    {
+      "epoch": 1.2723287671232877,
+      "grad_norm": 2.1316046714782715,
+      "learning_rate": 3.229082047116166e-06,
+      "loss": 1.7952,
+      "step": 1161
+    },
+    {
+      "epoch": 1.2734246575342465,
+      "grad_norm": 2.4078125953674316,
+      "learning_rate": 3.227051177904143e-06,
+      "loss": 1.869,
+      "step": 1162
+    },
+    {
+      "epoch": 1.2745205479452055,
+      "grad_norm": 2.363672971725464,
+      "learning_rate": 3.2250203086921204e-06,
+      "loss": 1.8478,
+      "step": 1163
+    },
+    {
+      "epoch": 1.2756164383561643,
+      "grad_norm": 2.240558385848999,
+      "learning_rate": 3.2229894394800977e-06,
+      "loss": 1.8483,
+      "step": 1164
+    },
+    {
+      "epoch": 1.2767123287671232,
+      "grad_norm": 2.9142000675201416,
+      "learning_rate": 3.220958570268075e-06,
+      "loss": 1.8789,
+      "step": 1165
+    },
+    {
+      "epoch": 1.2778082191780822,
+      "grad_norm": 2.1930558681488037,
+      "learning_rate": 3.218927701056052e-06,
+      "loss": 1.7659,
+      "step": 1166
+    },
+    {
+      "epoch": 1.2789041095890412,
+      "grad_norm": 2.312471866607666,
+      "learning_rate": 3.2168968318440295e-06,
+      "loss": 1.8374,
+      "step": 1167
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 2.6222236156463623,
+      "learning_rate": 3.2148659626320068e-06,
+      "loss": 1.7539,
+      "step": 1168
+    },
+    {
+      "epoch": 1.2810958904109588,
+      "grad_norm": 2.3013322353363037,
+      "learning_rate": 3.212835093419984e-06,
+      "loss": 1.9011,
+      "step": 1169
+    },
+    {
+      "epoch": 1.2821917808219179,
+      "grad_norm": 2.215240716934204,
+      "learning_rate": 3.2108042242079613e-06,
+      "loss": 1.8032,
+      "step": 1170
+    },
+    {
+      "epoch": 1.2821917808219179,
+      "eval_loss": 2.0365872383117676,
+      "eval_runtime": 36.8265,
+      "eval_samples_per_second": 27.154,
+      "eval_steps_per_second": 1.14,
+      "step": 1170
+    },
+    {
+      "epoch": 1.2832876712328767,
+      "grad_norm": 2.3620481491088867,
+      "learning_rate": 3.2087733549959386e-06,
+      "loss": 1.8258,
+      "step": 1171
+    },
+    {
+      "epoch": 1.2843835616438355,
+      "grad_norm": 2.2650208473205566,
+      "learning_rate": 3.206742485783916e-06,
+      "loss": 1.8888,
+      "step": 1172
+    },
+    {
+      "epoch": 1.2854794520547945,
+      "grad_norm": 2.2564077377319336,
+      "learning_rate": 3.2047116165718927e-06,
+      "loss": 1.7874,
+      "step": 1173
+    },
+    {
+      "epoch": 1.2865753424657536,
+      "grad_norm": 2.2855281829833984,
+      "learning_rate": 3.2026807473598704e-06,
+      "loss": 1.8003,
+      "step": 1174
+    },
+    {
+      "epoch": 1.2876712328767124,
+      "grad_norm": 2.353893756866455,
+      "learning_rate": 3.2006498781478473e-06,
+      "loss": 1.8466,
+      "step": 1175
+    },
+    {
+      "epoch": 1.2887671232876712,
+      "grad_norm": 2.1893045902252197,
+      "learning_rate": 3.198619008935825e-06,
+      "loss": 1.8652,
+      "step": 1176
+    },
+    {
+      "epoch": 1.2898630136986302,
+      "grad_norm": 2.6629793643951416,
+      "learning_rate": 3.196588139723802e-06,
+      "loss": 1.797,
+      "step": 1177
+    },
+    {
+      "epoch": 1.290958904109589,
+      "grad_norm": 2.2427361011505127,
+      "learning_rate": 3.1945572705117795e-06,
+      "loss": 1.8294,
+      "step": 1178
+    },
+    {
+      "epoch": 1.2920547945205478,
+      "grad_norm": 2.321878671646118,
+      "learning_rate": 3.1925264012997564e-06,
+      "loss": 1.8494,
+      "step": 1179
+    },
+    {
+      "epoch": 1.2931506849315069,
+      "grad_norm": 2.6928281784057617,
+      "learning_rate": 3.190495532087734e-06,
+      "loss": 1.8574,
+      "step": 1180
+    },
+    {
+      "epoch": 1.2931506849315069,
+      "eval_loss": 2.0426652431488037,
+      "eval_runtime": 36.8482,
+      "eval_samples_per_second": 27.138,
+      "eval_steps_per_second": 1.14,
+      "step": 1180
+    },
+    {
+      "epoch": 1.2942465753424657,
+      "grad_norm": 2.382164239883423,
+      "learning_rate": 3.188464662875711e-06,
+      "loss": 1.8518,
+      "step": 1181
+    },
+    {
+      "epoch": 1.2953424657534247,
+      "grad_norm": 2.4109256267547607,
+      "learning_rate": 3.1864337936636886e-06,
+      "loss": 1.9083,
+      "step": 1182
+    },
+    {
+      "epoch": 1.2964383561643835,
+      "grad_norm": 2.468062400817871,
+      "learning_rate": 3.1844029244516655e-06,
+      "loss": 1.8247,
+      "step": 1183
+    },
+    {
+      "epoch": 1.2975342465753426,
+      "grad_norm": 2.459378242492676,
+      "learning_rate": 3.182372055239643e-06,
+      "loss": 1.791,
+      "step": 1184
+    },
+    {
+      "epoch": 1.2986301369863014,
+      "grad_norm": 2.3331072330474854,
+      "learning_rate": 3.18034118602762e-06,
+      "loss": 1.7869,
+      "step": 1185
+    },
+    {
+      "epoch": 1.2997260273972602,
+      "grad_norm": 2.339428424835205,
+      "learning_rate": 3.1783103168155973e-06,
+      "loss": 1.8729,
+      "step": 1186
+    },
+    {
+      "epoch": 1.3008219178082192,
+      "grad_norm": 2.3033406734466553,
+      "learning_rate": 3.1762794476035746e-06,
+      "loss": 1.8455,
+      "step": 1187
+    },
+    {
+      "epoch": 1.301917808219178,
+      "grad_norm": 2.2480714321136475,
+      "learning_rate": 3.174248578391552e-06,
+      "loss": 1.8233,
+      "step": 1188
+    },
+    {
+      "epoch": 1.303013698630137,
+      "grad_norm": 2.2953624725341797,
+      "learning_rate": 3.172217709179529e-06,
+      "loss": 1.8261,
+      "step": 1189
+    },
+    {
+      "epoch": 1.3041095890410959,
+      "grad_norm": 2.494307518005371,
+      "learning_rate": 3.1701868399675064e-06,
+      "loss": 1.8153,
+      "step": 1190
+    },
+    {
+      "epoch": 1.3041095890410959,
+      "eval_loss": 2.037719964981079,
+      "eval_runtime": 36.9788,
+      "eval_samples_per_second": 27.043,
+      "eval_steps_per_second": 1.136,
+      "step": 1190
+    },
+    {
+      "epoch": 1.305205479452055,
+      "grad_norm": 2.314159631729126,
+      "learning_rate": 3.1681559707554837e-06,
+      "loss": 1.8406,
+      "step": 1191
+    },
+    {
+      "epoch": 1.3063013698630137,
+      "grad_norm": 2.4262588024139404,
+      "learning_rate": 3.166125101543461e-06,
+      "loss": 1.7803,
+      "step": 1192
+    },
+    {
+      "epoch": 1.3073972602739725,
+      "grad_norm": 2.3800835609436035,
+      "learning_rate": 3.1640942323314382e-06,
+      "loss": 1.8521,
+      "step": 1193
+    },
+    {
+      "epoch": 1.3084931506849315,
+      "grad_norm": 2.234551429748535,
+      "learning_rate": 3.1620633631194155e-06,
+      "loss": 1.845,
+      "step": 1194
+    },
+    {
+      "epoch": 1.3095890410958904,
+      "grad_norm": 2.167635440826416,
+      "learning_rate": 3.1600324939073928e-06,
+      "loss": 1.789,
+      "step": 1195
+    },
+    {
+      "epoch": 1.3106849315068494,
+      "grad_norm": 2.320247173309326,
+      "learning_rate": 3.15800162469537e-06,
+      "loss": 1.8311,
+      "step": 1196
+    },
+    {
+      "epoch": 1.3117808219178082,
+      "grad_norm": 2.2715561389923096,
+      "learning_rate": 3.155970755483347e-06,
+      "loss": 1.8308,
+      "step": 1197
+    },
+    {
+      "epoch": 1.3128767123287672,
+      "grad_norm": 2.2535455226898193,
+      "learning_rate": 3.1539398862713246e-06,
+      "loss": 1.8358,
+      "step": 1198
+    },
+    {
+      "epoch": 1.313972602739726,
+      "grad_norm": 2.2399163246154785,
+      "learning_rate": 3.1519090170593015e-06,
+      "loss": 1.8513,
+      "step": 1199
+    },
+    {
+      "epoch": 1.3150684931506849,
+      "grad_norm": 2.2984132766723633,
+      "learning_rate": 3.149878147847279e-06,
+      "loss": 1.8377,
+      "step": 1200
+    },
+    {
+      "epoch": 1.3150684931506849,
+      "eval_loss": 2.0334064960479736,
+      "eval_runtime": 37.0346,
+      "eval_samples_per_second": 27.002,
+      "eval_steps_per_second": 1.134,
+      "step": 1200
+    },
+    {
+      "epoch": 1.316164383561644,
+      "grad_norm": 2.1296892166137695,
+      "learning_rate": 3.147847278635256e-06,
+      "loss": 1.9023,
+      "step": 1201
+    },
+    {
+      "epoch": 1.3172602739726027,
+      "grad_norm": 2.1034350395202637,
+      "learning_rate": 3.1458164094232337e-06,
+      "loss": 1.917,
+      "step": 1202
+    },
+    {
+      "epoch": 1.3183561643835615,
+      "grad_norm": 2.258863687515259,
+      "learning_rate": 3.1437855402112106e-06,
+      "loss": 1.8093,
+      "step": 1203
+    },
+    {
+      "epoch": 1.3194520547945205,
+      "grad_norm": 2.20119309425354,
+      "learning_rate": 3.1417546709991883e-06,
+      "loss": 1.7796,
+      "step": 1204
+    },
+    {
+      "epoch": 1.3205479452054796,
+      "grad_norm": 2.3005869388580322,
+      "learning_rate": 3.139723801787165e-06,
+      "loss": 1.8496,
+      "step": 1205
+    },
+    {
+      "epoch": 1.3216438356164384,
+      "grad_norm": 2.2009031772613525,
+      "learning_rate": 3.137692932575143e-06,
+      "loss": 1.8244,
+      "step": 1206
+    },
+    {
+      "epoch": 1.3227397260273972,
+      "grad_norm": 2.3467633724212646,
+      "learning_rate": 3.1356620633631197e-06,
+      "loss": 1.8803,
+      "step": 1207
+    },
+    {
+      "epoch": 1.3238356164383562,
+      "grad_norm": 2.318711519241333,
+      "learning_rate": 3.1336311941510974e-06,
+      "loss": 1.7999,
+      "step": 1208
+    },
+    {
+      "epoch": 1.324931506849315,
+      "grad_norm": 2.182602882385254,
+      "learning_rate": 3.1316003249390742e-06,
+      "loss": 1.836,
+      "step": 1209
+    },
+    {
+      "epoch": 1.3260273972602739,
+      "grad_norm": 2.217298984527588,
+      "learning_rate": 3.129569455727051e-06,
+      "loss": 1.8776,
+      "step": 1210
+    },
+    {
+      "epoch": 1.3260273972602739,
+      "eval_loss": 2.033573627471924,
+      "eval_runtime": 36.8029,
+      "eval_samples_per_second": 27.172,
+      "eval_steps_per_second": 1.141,
+      "step": 1210
+    },
+    {
+      "epoch": 1.3271232876712329,
+      "grad_norm": 2.2814879417419434,
+      "learning_rate": 3.1275385865150288e-06,
+      "loss": 1.8739,
+      "step": 1211
+    },
+    {
+      "epoch": 1.3282191780821917,
+      "grad_norm": 2.1692471504211426,
+      "learning_rate": 3.1255077173030056e-06,
+      "loss": 1.8539,
+      "step": 1212
+    },
+    {
+      "epoch": 1.3293150684931507,
+      "grad_norm": 2.2472000122070312,
+      "learning_rate": 3.1234768480909833e-06,
+      "loss": 1.8527,
+      "step": 1213
+    },
+    {
+      "epoch": 1.3304109589041095,
+      "grad_norm": 2.1969852447509766,
+      "learning_rate": 3.12144597887896e-06,
+      "loss": 1.8429,
+      "step": 1214
+    },
+    {
+      "epoch": 1.3315068493150686,
+      "grad_norm": 2.141629219055176,
+      "learning_rate": 3.119415109666938e-06,
+      "loss": 1.868,
+      "step": 1215
+    },
+    {
+      "epoch": 1.3326027397260274,
+      "grad_norm": 2.282562494277954,
+      "learning_rate": 3.1173842404549147e-06,
+      "loss": 1.8913,
+      "step": 1216
+    },
+    {
+      "epoch": 1.3336986301369862,
+      "grad_norm": 2.2375850677490234,
+      "learning_rate": 3.1153533712428924e-06,
+      "loss": 1.8462,
+      "step": 1217
+    },
+    {
+      "epoch": 1.3347945205479452,
+      "grad_norm": 2.3529837131500244,
+      "learning_rate": 3.1133225020308693e-06,
+      "loss": 1.8445,
+      "step": 1218
+    },
+    {
+      "epoch": 1.335890410958904,
+      "grad_norm": 2.385857343673706,
+      "learning_rate": 3.111291632818847e-06,
+      "loss": 1.89,
+      "step": 1219
+    },
+    {
+      "epoch": 1.336986301369863,
+      "grad_norm": 2.301253080368042,
+      "learning_rate": 3.109260763606824e-06,
+      "loss": 1.8882,
+      "step": 1220
+    },
+    {
+      "epoch": 1.336986301369863,
+      "eval_loss": 2.040025234222412,
+      "eval_runtime": 36.839,
+      "eval_samples_per_second": 27.145,
+      "eval_steps_per_second": 1.14,
+      "step": 1220
+    },
+    {
+      "epoch": 1.3380821917808219,
+      "grad_norm": 2.3772335052490234,
+      "learning_rate": 3.1072298943948015e-06,
+      "loss": 1.8545,
+      "step": 1221
+    },
+    {
+      "epoch": 1.339178082191781,
+      "grad_norm": 2.387830972671509,
+      "learning_rate": 3.1051990251827784e-06,
+      "loss": 1.8653,
+      "step": 1222
+    },
+    {
+      "epoch": 1.3402739726027397,
+      "grad_norm": 2.4285190105438232,
+      "learning_rate": 3.1031681559707557e-06,
+      "loss": 1.8268,
+      "step": 1223
+    },
+    {
+      "epoch": 1.3413698630136985,
+      "grad_norm": 2.1892075538635254,
+      "learning_rate": 3.101137286758733e-06,
+      "loss": 1.8818,
+      "step": 1224
+    },
+    {
+      "epoch": 1.3424657534246576,
+      "grad_norm": 2.2510695457458496,
+      "learning_rate": 3.09910641754671e-06,
+      "loss": 1.879,
+      "step": 1225
+    },
+    {
+      "epoch": 1.3435616438356164,
+      "grad_norm": 2.505671977996826,
+      "learning_rate": 3.0970755483346875e-06,
+      "loss": 1.872,
+      "step": 1226
+    },
+    {
+      "epoch": 1.3446575342465754,
+      "grad_norm": 2.5315067768096924,
+      "learning_rate": 3.0950446791226648e-06,
+      "loss": 1.8651,
+      "step": 1227
+    },
+    {
+      "epoch": 1.3457534246575342,
+      "grad_norm": 2.223600387573242,
+      "learning_rate": 3.093013809910642e-06,
+      "loss": 1.9167,
+      "step": 1228
+    },
+    {
+      "epoch": 1.3468493150684933,
+      "grad_norm": 2.357802391052246,
+      "learning_rate": 3.0909829406986193e-06,
+      "loss": 1.8593,
+      "step": 1229
+    },
+    {
+      "epoch": 1.347945205479452,
+      "grad_norm": 2.3102364540100098,
+      "learning_rate": 3.0889520714865966e-06,
+      "loss": 1.8994,
+      "step": 1230
+    },
+    {
+      "epoch": 1.347945205479452,
+      "eval_loss": 2.0332930088043213,
+      "eval_runtime": 36.989,
+      "eval_samples_per_second": 27.035,
+      "eval_steps_per_second": 1.135,
+      "step": 1230
+    },
+    {
+      "epoch": 1.3490410958904109,
+      "grad_norm": 2.085080623626709,
+      "learning_rate": 3.086921202274574e-06,
+      "loss": 1.8741,
+      "step": 1231
+    },
+    {
+      "epoch": 1.35013698630137,
+      "grad_norm": 2.267305612564087,
+      "learning_rate": 3.084890333062551e-06,
+      "loss": 1.8426,
+      "step": 1232
+    },
+    {
+      "epoch": 1.3512328767123287,
+      "grad_norm": 2.181033134460449,
+      "learning_rate": 3.0828594638505284e-06,
+      "loss": 1.8199,
+      "step": 1233
+    },
+    {
+      "epoch": 1.3523287671232875,
+      "grad_norm": 2.288386821746826,
+      "learning_rate": 3.0808285946385057e-06,
+      "loss": 1.8469,
+      "step": 1234
+    },
+    {
+      "epoch": 1.3534246575342466,
+      "grad_norm": 2.1516127586364746,
+      "learning_rate": 3.078797725426483e-06,
+      "loss": 1.8056,
+      "step": 1235
+    },
+    {
+      "epoch": 1.3545205479452056,
+      "grad_norm": 2.2699198722839355,
+      "learning_rate": 3.07676685621446e-06,
+      "loss": 1.7535,
+      "step": 1236
+    },
+    {
+      "epoch": 1.3556164383561644,
+      "grad_norm": 2.193251132965088,
+      "learning_rate": 3.0747359870024375e-06,
+      "loss": 1.8407,
+      "step": 1237
+    },
+    {
+      "epoch": 1.3567123287671232,
+      "grad_norm": 2.2217774391174316,
+      "learning_rate": 3.0727051177904144e-06,
+      "loss": 1.8062,
+      "step": 1238
+    },
+    {
+      "epoch": 1.3578082191780823,
+      "grad_norm": 2.2376203536987305,
+      "learning_rate": 3.070674248578392e-06,
+      "loss": 1.829,
+      "step": 1239
+    },
+    {
+      "epoch": 1.358904109589041,
+      "grad_norm": 2.387559413909912,
+      "learning_rate": 3.068643379366369e-06,
+      "loss": 1.8459,
+      "step": 1240
+    },
+    {
+      "epoch": 1.358904109589041,
+      "eval_loss": 2.029298782348633,
+      "eval_runtime": 37.0491,
+      "eval_samples_per_second": 26.991,
+      "eval_steps_per_second": 1.134,
+      "step": 1240
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 2.1940178871154785,
+      "learning_rate": 3.0666125101543466e-06,
+      "loss": 1.831,
+      "step": 1241
+    },
+    {
+      "epoch": 1.361095890410959,
+      "grad_norm": 2.145514488220215,
+      "learning_rate": 3.0645816409423235e-06,
+      "loss": 1.7919,
+      "step": 1242
+    },
+    {
+      "epoch": 1.362191780821918,
+      "grad_norm": 2.319164276123047,
+      "learning_rate": 3.062550771730301e-06,
+      "loss": 1.8472,
+      "step": 1243
+    },
+    {
+      "epoch": 1.3632876712328768,
+      "grad_norm": 2.218986749649048,
+      "learning_rate": 3.060519902518278e-06,
+      "loss": 1.8184,
+      "step": 1244
+    },
+    {
+      "epoch": 1.3643835616438356,
+      "grad_norm": 2.3755338191986084,
+      "learning_rate": 3.0584890333062557e-06,
+      "loss": 1.8166,
+      "step": 1245
+    },
+    {
+      "epoch": 1.3654794520547946,
+      "grad_norm": 2.3186213970184326,
+      "learning_rate": 3.0564581640942326e-06,
+      "loss": 1.7599,
+      "step": 1246
+    },
+    {
+      "epoch": 1.3665753424657534,
+      "grad_norm": 2.193958282470703,
+      "learning_rate": 3.0544272948822094e-06,
+      "loss": 1.8404,
+      "step": 1247
+    },
+    {
+      "epoch": 1.3676712328767122,
+      "grad_norm": 2.40419602394104,
+      "learning_rate": 3.052396425670187e-06,
+      "loss": 1.821,
+      "step": 1248
+    },
+    {
+      "epoch": 1.3687671232876713,
+      "grad_norm": 2.445542573928833,
+      "learning_rate": 3.050365556458164e-06,
+      "loss": 1.8518,
+      "step": 1249
+    },
+    {
+      "epoch": 1.36986301369863,
+      "grad_norm": 2.2776567935943604,
+      "learning_rate": 3.0483346872461417e-06,
+      "loss": 1.8121,
+      "step": 1250
+    },
+    {
+      "epoch": 1.36986301369863,
+      "eval_loss": 2.030043125152588,
+      "eval_runtime": 37.0721,
+      "eval_samples_per_second": 26.974,
+      "eval_steps_per_second": 1.133,
+      "step": 1250
+    },
+    {
+      "epoch": 1.370958904109589,
+      "grad_norm": 2.420448064804077,
+      "learning_rate": 3.0463038180341185e-06,
+      "loss": 1.8168,
+      "step": 1251
+    },
+    {
+      "epoch": 1.372054794520548,
+      "grad_norm": 2.252065658569336,
+      "learning_rate": 3.0442729488220962e-06,
+      "loss": 1.8757,
+      "step": 1252
+    },
+    {
+      "epoch": 1.373150684931507,
+      "grad_norm": 2.4772560596466064,
+      "learning_rate": 3.042242079610073e-06,
+      "loss": 1.8711,
+      "step": 1253
+    },
+    {
+      "epoch": 1.3742465753424657,
+      "grad_norm": 2.3986129760742188,
+      "learning_rate": 3.0402112103980508e-06,
+      "loss": 1.8636,
+      "step": 1254
+    },
+    {
+      "epoch": 1.3753424657534246,
+      "grad_norm": 2.257274866104126,
+      "learning_rate": 3.0381803411860276e-06,
+      "loss": 1.7858,
+      "step": 1255
+    },
+    {
+      "epoch": 1.3764383561643836,
+      "grad_norm": 2.2373499870300293,
+      "learning_rate": 3.0361494719740053e-06,
+      "loss": 1.817,
+      "step": 1256
+    },
+    {
+      "epoch": 1.3775342465753424,
+      "grad_norm": 2.292388916015625,
+      "learning_rate": 3.034118602761982e-06,
+      "loss": 1.827,
+      "step": 1257
+    },
+    {
+      "epoch": 1.3786301369863014,
+      "grad_norm": 2.4618642330169678,
+      "learning_rate": 3.03208773354996e-06,
+      "loss": 1.8218,
+      "step": 1258
+    },
+    {
+      "epoch": 1.3797260273972602,
+      "grad_norm": 2.271958112716675,
+      "learning_rate": 3.0300568643379367e-06,
+      "loss": 1.8092,
+      "step": 1259
+    },
+    {
+      "epoch": 1.3808219178082193,
+      "grad_norm": 2.3990538120269775,
+      "learning_rate": 3.028025995125914e-06,
+      "loss": 1.8633,
+      "step": 1260
+    },
+    {
+      "epoch": 1.3808219178082193,
+      "eval_loss": 2.028458595275879,
+      "eval_runtime": 37.0659,
+      "eval_samples_per_second": 26.979,
+      "eval_steps_per_second": 1.133,
+      "step": 1260
+    },
+    {
+      "epoch": 1.381917808219178,
+      "grad_norm": 2.208803653717041,
+      "learning_rate": 3.0259951259138913e-06,
+      "loss": 1.8463,
+      "step": 1261
+    },
+    {
+      "epoch": 1.383013698630137,
+      "grad_norm": 2.3269028663635254,
+      "learning_rate": 3.0239642567018686e-06,
+      "loss": 1.7784,
+      "step": 1262
+    },
+    {
+      "epoch": 1.384109589041096,
+      "grad_norm": 2.367910623550415,
+      "learning_rate": 3.021933387489846e-06,
+      "loss": 1.8281,
+      "step": 1263
+    },
+    {
+      "epoch": 1.3852054794520547,
+      "grad_norm": 2.273686647415161,
+      "learning_rate": 3.019902518277823e-06,
+      "loss": 1.7832,
+      "step": 1264
+    },
+    {
+      "epoch": 1.3863013698630138,
+      "grad_norm": 2.421964168548584,
+      "learning_rate": 3.0178716490658004e-06,
+      "loss": 1.8407,
+      "step": 1265
+    },
+    {
+      "epoch": 1.3873972602739726,
+      "grad_norm": 2.2627696990966797,
+      "learning_rate": 3.0158407798537777e-06,
+      "loss": 1.8718,
+      "step": 1266
+    },
+    {
+      "epoch": 1.3884931506849316,
+      "grad_norm": 2.1981618404388428,
+      "learning_rate": 3.013809910641755e-06,
+      "loss": 1.7763,
+      "step": 1267
+    },
+    {
+      "epoch": 1.3895890410958904,
+      "grad_norm": 2.245253562927246,
+      "learning_rate": 3.0117790414297322e-06,
+      "loss": 1.7915,
+      "step": 1268
+    },
+    {
+      "epoch": 1.3906849315068492,
+      "grad_norm": 2.2292444705963135,
+      "learning_rate": 3.0097481722177095e-06,
+      "loss": 1.7909,
+      "step": 1269
+    },
+    {
+      "epoch": 1.3917808219178083,
+      "grad_norm": 2.2290430068969727,
+      "learning_rate": 3.0077173030056868e-06,
+      "loss": 1.8364,
+      "step": 1270
+    },
+    {
+      "epoch": 1.3917808219178083,
+      "eval_loss": 2.0290327072143555,
+      "eval_runtime": 37.0666,
+      "eval_samples_per_second": 26.978,
+      "eval_steps_per_second": 1.133,
+      "step": 1270
+    },
+    {
+      "epoch": 1.392876712328767,
+      "grad_norm": 2.220881700515747,
+      "learning_rate": 3.005686433793664e-06,
+      "loss": 1.7968,
+      "step": 1271
+    },
+    {
+      "epoch": 1.393972602739726,
+      "grad_norm": 2.375332832336426,
+      "learning_rate": 3.0036555645816413e-06,
+      "loss": 1.8502,
+      "step": 1272
+    },
+    {
+      "epoch": 1.395068493150685,
+      "grad_norm": 2.3065083026885986,
+      "learning_rate": 3.001624695369618e-06,
+      "loss": 1.8116,
+      "step": 1273
+    },
+    {
+      "epoch": 1.396164383561644,
+      "grad_norm": 2.7262511253356934,
+      "learning_rate": 2.999593826157596e-06,
+      "loss": 1.8641,
+      "step": 1274
+    },
+    {
+      "epoch": 1.3972602739726028,
+      "grad_norm": 2.3381216526031494,
+      "learning_rate": 2.9975629569455727e-06,
+      "loss": 1.8128,
+      "step": 1275
+    },
+    {
+      "epoch": 1.3983561643835616,
+      "grad_norm": 2.1994192600250244,
+      "learning_rate": 2.9955320877335504e-06,
+      "loss": 1.8563,
+      "step": 1276
+    },
+    {
+      "epoch": 1.3994520547945206,
+      "grad_norm": 2.3810958862304688,
+      "learning_rate": 2.9935012185215273e-06,
+      "loss": 1.8196,
+      "step": 1277
+    },
+    {
+      "epoch": 1.4005479452054794,
+      "grad_norm": 2.4343955516815186,
+      "learning_rate": 2.991470349309505e-06,
+      "loss": 1.8564,
+      "step": 1278
+    },
+    {
+      "epoch": 1.4016438356164382,
+      "grad_norm": 2.1929988861083984,
+      "learning_rate": 2.989439480097482e-06,
+      "loss": 1.8119,
+      "step": 1279
+    },
+    {
+      "epoch": 1.4027397260273973,
+      "grad_norm": 2.2305309772491455,
+      "learning_rate": 2.9874086108854595e-06,
+      "loss": 1.8289,
+      "step": 1280
+    },
+    {
+      "epoch": 1.4027397260273973,
+      "eval_loss": 2.032022476196289,
+      "eval_runtime": 37.084,
+      "eval_samples_per_second": 26.966,
+      "eval_steps_per_second": 1.133,
+      "step": 1280
+    },
+    {
+      "epoch": 1.403835616438356,
+      "grad_norm": 2.6255314350128174,
+      "learning_rate": 2.9853777416734364e-06,
+      "loss": 1.8176,
+      "step": 1281
+    },
+    {
+      "epoch": 1.4049315068493151,
+      "grad_norm": 2.238891363143921,
+      "learning_rate": 2.983346872461414e-06,
+      "loss": 1.8679,
+      "step": 1282
+    },
+    {
+      "epoch": 1.406027397260274,
+      "grad_norm": 2.9972007274627686,
+      "learning_rate": 2.981316003249391e-06,
+      "loss": 1.8839,
+      "step": 1283
+    },
+    {
+      "epoch": 1.407123287671233,
+      "grad_norm": 2.3255932331085205,
+      "learning_rate": 2.9792851340373678e-06,
+      "loss": 1.8451,
+      "step": 1284
+    },
+    {
+      "epoch": 1.4082191780821918,
+      "grad_norm": 2.290395498275757,
+      "learning_rate": 2.9772542648253455e-06,
+      "loss": 1.8244,
+      "step": 1285
+    },
+    {
+      "epoch": 1.4093150684931506,
+      "grad_norm": 2.288728713989258,
+      "learning_rate": 2.9752233956133223e-06,
+      "loss": 1.8628,
+      "step": 1286
+    },
+    {
+      "epoch": 1.4104109589041096,
+      "grad_norm": 2.177381992340088,
+      "learning_rate": 2.9731925264013e-06,
+      "loss": 1.8111,
+      "step": 1287
+    },
+    {
+      "epoch": 1.4115068493150684,
+      "grad_norm": 2.214956760406494,
+      "learning_rate": 2.971161657189277e-06,
+      "loss": 1.8377,
+      "step": 1288
+    },
+    {
+      "epoch": 1.4126027397260275,
+      "grad_norm": 2.3859663009643555,
+      "learning_rate": 2.9691307879772546e-06,
+      "loss": 1.8154,
+      "step": 1289
+    },
+    {
+      "epoch": 1.4136986301369863,
+      "grad_norm": 2.0873706340789795,
+      "learning_rate": 2.9670999187652314e-06,
+      "loss": 1.8098,
+      "step": 1290
+    },
+    {
+      "epoch": 1.4136986301369863,
+      "eval_loss": 2.029996633529663,
+      "eval_runtime": 37.0979,
+      "eval_samples_per_second": 26.956,
+      "eval_steps_per_second": 1.132,
+      "step": 1290
+    },
+    {
+      "epoch": 1.4147945205479453,
+      "grad_norm": 2.2632675170898438,
+      "learning_rate": 2.965069049553209e-06,
+      "loss": 1.7974,
+      "step": 1291
+    },
+    {
+      "epoch": 1.415890410958904,
+      "grad_norm": 2.208515167236328,
+      "learning_rate": 2.963038180341186e-06,
+      "loss": 1.8856,
+      "step": 1292
+    },
+    {
+      "epoch": 1.416986301369863,
+      "grad_norm": 2.176525115966797,
+      "learning_rate": 2.9610073111291637e-06,
+      "loss": 1.7783,
+      "step": 1293
+    },
+    {
+      "epoch": 1.418082191780822,
+      "grad_norm": 2.226682186126709,
+      "learning_rate": 2.9589764419171405e-06,
+      "loss": 1.865,
+      "step": 1294
+    },
+    {
+      "epoch": 1.4191780821917808,
+      "grad_norm": 2.3544883728027344,
+      "learning_rate": 2.9569455727051182e-06,
+      "loss": 1.8808,
+      "step": 1295
+    },
+    {
+      "epoch": 1.4202739726027398,
+      "grad_norm": 2.1729962825775146,
+      "learning_rate": 2.954914703493095e-06,
+      "loss": 1.7927,
+      "step": 1296
+    },
+    {
+      "epoch": 1.4213698630136986,
+      "grad_norm": 2.168168067932129,
+      "learning_rate": 2.9528838342810724e-06,
+      "loss": 1.8253,
+      "step": 1297
+    },
+    {
+      "epoch": 1.4224657534246576,
+      "grad_norm": 2.2479910850524902,
+      "learning_rate": 2.9508529650690496e-06,
+      "loss": 1.7976,
+      "step": 1298
+    },
+    {
+      "epoch": 1.4235616438356165,
+      "grad_norm": 2.1733837127685547,
+      "learning_rate": 2.948822095857027e-06,
+      "loss": 1.8015,
+      "step": 1299
+    },
+    {
+      "epoch": 1.4246575342465753,
+      "grad_norm": 2.2553391456604004,
+      "learning_rate": 2.946791226645004e-06,
+      "loss": 1.8545,
+      "step": 1300
+    },
+    {
+      "epoch": 1.4246575342465753,
+      "eval_loss": 2.0245048999786377,
+      "eval_runtime": 37.0943,
+      "eval_samples_per_second": 26.958,
+      "eval_steps_per_second": 1.132,
+      "step": 1300
+    },
+    {
+      "epoch": 1.4257534246575343,
+      "grad_norm": 2.2817769050598145,
+      "learning_rate": 2.9447603574329815e-06,
+      "loss": 1.8017,
+      "step": 1301
+    },
+    {
+      "epoch": 1.426849315068493,
+      "grad_norm": 2.170865297317505,
+      "learning_rate": 2.9427294882209587e-06,
+      "loss": 1.7738,
+      "step": 1302
+    },
+    {
+      "epoch": 1.427945205479452,
+      "grad_norm": 2.296919107437134,
+      "learning_rate": 2.940698619008936e-06,
+      "loss": 1.8098,
+      "step": 1303
+    },
+    {
+      "epoch": 1.429041095890411,
+      "grad_norm": 2.476266622543335,
+      "learning_rate": 2.9386677497969133e-06,
+      "loss": 1.8229,
+      "step": 1304
+    },
+    {
+      "epoch": 1.43013698630137,
+      "grad_norm": 2.400918483734131,
+      "learning_rate": 2.9366368805848906e-06,
+      "loss": 1.8057,
+      "step": 1305
+    },
+    {
+      "epoch": 1.4312328767123288,
+      "grad_norm": 2.393892765045166,
+      "learning_rate": 2.934606011372868e-06,
+      "loss": 1.8568,
+      "step": 1306
+    },
+    {
+      "epoch": 1.4323287671232876,
+      "grad_norm": 2.2551307678222656,
+      "learning_rate": 2.932575142160845e-06,
+      "loss": 1.8493,
+      "step": 1307
+    },
+    {
+      "epoch": 1.4334246575342466,
+      "grad_norm": 2.3091611862182617,
+      "learning_rate": 2.9305442729488224e-06,
+      "loss": 1.8297,
+      "step": 1308
+    },
+    {
+      "epoch": 1.4345205479452054,
+      "grad_norm": 2.231614828109741,
+      "learning_rate": 2.9285134037367997e-06,
+      "loss": 1.8345,
+      "step": 1309
+    },
+    {
+      "epoch": 1.4356164383561643,
+      "grad_norm": 2.24942946434021,
+      "learning_rate": 2.9264825345247765e-06,
+      "loss": 1.8284,
+      "step": 1310
+    },
+    {
+      "epoch": 1.4356164383561643,
+      "eval_loss": 2.024843215942383,
+      "eval_runtime": 36.7795,
+      "eval_samples_per_second": 27.189,
+      "eval_steps_per_second": 1.142,
+      "step": 1310
+    },
+    {
+      "epoch": 1.4367123287671233,
+      "grad_norm": 2.2798643112182617,
+      "learning_rate": 2.9244516653127542e-06,
+      "loss": 1.8554,
+      "step": 1311
+    },
+    {
+      "epoch": 1.4378082191780823,
+      "grad_norm": 2.342576265335083,
+      "learning_rate": 2.922420796100731e-06,
+      "loss": 1.8296,
+      "step": 1312
+    },
+    {
+      "epoch": 1.4389041095890411,
+      "grad_norm": 2.268016815185547,
+      "learning_rate": 2.9203899268887088e-06,
+      "loss": 1.7467,
+      "step": 1313
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 2.2520217895507812,
+      "learning_rate": 2.9183590576766856e-06,
+      "loss": 1.8748,
+      "step": 1314
+    },
+    {
+      "epoch": 1.441095890410959,
+      "grad_norm": 2.2091598510742188,
+      "learning_rate": 2.9163281884646633e-06,
+      "loss": 1.7715,
+      "step": 1315
+    },
+    {
+      "epoch": 1.4421917808219178,
+      "grad_norm": 2.197575092315674,
+      "learning_rate": 2.91429731925264e-06,
+      "loss": 1.8265,
+      "step": 1316
+    },
+    {
+      "epoch": 1.4432876712328766,
+      "grad_norm": 2.261122465133667,
+      "learning_rate": 2.912266450040618e-06,
+      "loss": 1.8758,
+      "step": 1317
+    },
+    {
+      "epoch": 1.4443835616438356,
+      "grad_norm": 2.3291778564453125,
+      "learning_rate": 2.9102355808285947e-06,
+      "loss": 1.8213,
+      "step": 1318
+    },
+    {
+      "epoch": 1.4454794520547944,
+      "grad_norm": 2.2182250022888184,
+      "learning_rate": 2.9082047116165724e-06,
+      "loss": 1.822,
+      "step": 1319
+    },
+    {
+      "epoch": 1.4465753424657535,
+      "grad_norm": 2.295978546142578,
+      "learning_rate": 2.9061738424045493e-06,
+      "loss": 1.8008,
+      "step": 1320
+    },
+    {
+      "epoch": 1.4465753424657535,
+      "eval_loss": 2.0230674743652344,
+      "eval_runtime": 37.0013,
+      "eval_samples_per_second": 27.026,
+      "eval_steps_per_second": 1.135,
+      "step": 1320
+    },
+    {
+      "epoch": 1.4476712328767123,
+      "grad_norm": 2.3187007904052734,
+      "learning_rate": 2.904142973192527e-06,
+      "loss": 1.7923,
+      "step": 1321
+    },
+    {
+      "epoch": 1.4487671232876713,
+      "grad_norm": 2.242851972579956,
+      "learning_rate": 2.902112103980504e-06,
+      "loss": 1.8874,
+      "step": 1322
+    },
+    {
+      "epoch": 1.4498630136986301,
+      "grad_norm": 2.2560081481933594,
+      "learning_rate": 2.900081234768481e-06,
+      "loss": 1.8284,
+      "step": 1323
+    },
+    {
+      "epoch": 1.450958904109589,
+      "grad_norm": 2.3017799854278564,
+      "learning_rate": 2.8980503655564584e-06,
+      "loss": 1.8263,
+      "step": 1324
+    },
+    {
+      "epoch": 1.452054794520548,
+      "grad_norm": 2.3016278743743896,
+      "learning_rate": 2.8960194963444357e-06,
+      "loss": 1.8174,
+      "step": 1325
+    },
+    {
+      "epoch": 1.4531506849315068,
+      "grad_norm": 2.1965372562408447,
+      "learning_rate": 2.893988627132413e-06,
+      "loss": 1.7812,
+      "step": 1326
+    },
+    {
+      "epoch": 1.4542465753424658,
+      "grad_norm": 2.302969455718994,
+      "learning_rate": 2.8919577579203902e-06,
+      "loss": 1.8115,
+      "step": 1327
+    },
+    {
+      "epoch": 1.4553424657534246,
+      "grad_norm": 2.338066339492798,
+      "learning_rate": 2.8899268887083675e-06,
+      "loss": 1.7535,
+      "step": 1328
+    },
+    {
+      "epoch": 1.4564383561643837,
+      "grad_norm": 2.446503162384033,
+      "learning_rate": 2.8878960194963448e-06,
+      "loss": 1.8052,
+      "step": 1329
+    },
+    {
+      "epoch": 1.4575342465753425,
+      "grad_norm": 2.2319273948669434,
+      "learning_rate": 2.885865150284322e-06,
+      "loss": 1.8226,
+      "step": 1330
+    },
+    {
+      "epoch": 1.4575342465753425,
+      "eval_loss": 2.022364854812622,
+      "eval_runtime": 37.0901,
+      "eval_samples_per_second": 26.961,
+      "eval_steps_per_second": 1.132,
+      "step": 1330
+    },
+    {
+      "epoch": 1.4586301369863013,
+      "grad_norm": 2.391080856323242,
+      "learning_rate": 2.8838342810722993e-06,
+      "loss": 1.8529,
+      "step": 1331
+    },
+    {
+      "epoch": 1.4597260273972603,
+      "grad_norm": 2.274463415145874,
+      "learning_rate": 2.8818034118602766e-06,
+      "loss": 1.8043,
+      "step": 1332
+    },
+    {
+      "epoch": 1.4608219178082191,
+      "grad_norm": 2.4040963649749756,
+      "learning_rate": 2.879772542648254e-06,
+      "loss": 1.7842,
+      "step": 1333
+    },
+    {
+      "epoch": 1.4619178082191782,
+      "grad_norm": 2.268629312515259,
+      "learning_rate": 2.8777416734362307e-06,
+      "loss": 1.859,
+      "step": 1334
+    },
+    {
+      "epoch": 1.463013698630137,
+      "grad_norm": 2.4727842807769775,
+      "learning_rate": 2.8757108042242084e-06,
+      "loss": 1.7615,
+      "step": 1335
+    },
+    {
+      "epoch": 1.464109589041096,
+      "grad_norm": 2.179358959197998,
+      "learning_rate": 2.8736799350121853e-06,
+      "loss": 1.9085,
+      "step": 1336
+    },
+    {
+      "epoch": 1.4652054794520548,
+      "grad_norm": 2.220917224884033,
+      "learning_rate": 2.871649065800163e-06,
+      "loss": 1.8618,
+      "step": 1337
+    },
+    {
+      "epoch": 1.4663013698630136,
+      "grad_norm": 2.3807291984558105,
+      "learning_rate": 2.86961819658814e-06,
+      "loss": 1.88,
+      "step": 1338
+    },
+    {
+      "epoch": 1.4673972602739727,
+      "grad_norm": 2.272507667541504,
+      "learning_rate": 2.8675873273761175e-06,
+      "loss": 1.8045,
+      "step": 1339
+    },
+    {
+      "epoch": 1.4684931506849315,
+      "grad_norm": 2.1832377910614014,
+      "learning_rate": 2.8655564581640944e-06,
+      "loss": 1.8273,
+      "step": 1340
+    },
+    {
+      "epoch": 1.4684931506849315,
+      "eval_loss": 2.0218358039855957,
+      "eval_runtime": 37.1223,
+      "eval_samples_per_second": 26.938,
+      "eval_steps_per_second": 1.131,
+      "step": 1340
+    },
+    {
+      "epoch": 1.4695890410958903,
+      "grad_norm": 2.2655251026153564,
+      "learning_rate": 2.863525588952072e-06,
+      "loss": 1.8162,
+      "step": 1341
+    },
+    {
+      "epoch": 1.4706849315068493,
+      "grad_norm": 2.191852331161499,
+      "learning_rate": 2.861494719740049e-06,
+      "loss": 1.8778,
+      "step": 1342
+    },
+    {
+      "epoch": 1.4717808219178083,
+      "grad_norm": 2.1869187355041504,
+      "learning_rate": 2.8594638505280266e-06,
+      "loss": 1.7695,
+      "step": 1343
+    },
+    {
+      "epoch": 1.4728767123287672,
+      "grad_norm": 2.248108148574829,
+      "learning_rate": 2.8574329813160035e-06,
+      "loss": 1.7759,
+      "step": 1344
+    },
+    {
+      "epoch": 1.473972602739726,
+      "grad_norm": 2.2599704265594482,
+      "learning_rate": 2.855402112103981e-06,
+      "loss": 1.8214,
+      "step": 1345
+    },
+    {
+      "epoch": 1.475068493150685,
+      "grad_norm": 2.2920355796813965,
+      "learning_rate": 2.853371242891958e-06,
+      "loss": 1.8061,
+      "step": 1346
+    },
+    {
+      "epoch": 1.4761643835616438,
+      "grad_norm": 2.26715087890625,
+      "learning_rate": 2.851340373679935e-06,
+      "loss": 1.8244,
+      "step": 1347
+    },
+    {
+      "epoch": 1.4772602739726026,
+      "grad_norm": 2.349169969558716,
+      "learning_rate": 2.8493095044679126e-06,
+      "loss": 1.8401,
+      "step": 1348
+    },
+    {
+      "epoch": 1.4783561643835617,
+      "grad_norm": 2.3571698665618896,
+      "learning_rate": 2.8472786352558894e-06,
+      "loss": 1.8913,
+      "step": 1349
+    },
+    {
+      "epoch": 1.4794520547945205,
+      "grad_norm": 2.334057331085205,
+      "learning_rate": 2.845247766043867e-06,
+      "loss": 1.8434,
+      "step": 1350
+    },
+    {
+      "epoch": 1.4794520547945205,
+      "eval_loss": 2.0251200199127197,
+      "eval_runtime": 37.1298,
+      "eval_samples_per_second": 26.933,
+      "eval_steps_per_second": 1.131,
+      "step": 1350
+    },
+    {
+      "epoch": 1.4805479452054795,
+      "grad_norm": 2.363516330718994,
+      "learning_rate": 2.843216896831844e-06,
+      "loss": 1.7846,
+      "step": 1351
+    },
+    {
+      "epoch": 1.4816438356164383,
+      "grad_norm": 2.372403860092163,
+      "learning_rate": 2.8411860276198217e-06,
+      "loss": 1.7791,
+      "step": 1352
+    },
+    {
+      "epoch": 1.4827397260273973,
+      "grad_norm": 2.2666590213775635,
+      "learning_rate": 2.8391551584077985e-06,
+      "loss": 1.7969,
+      "step": 1353
+    },
+    {
+      "epoch": 1.4838356164383562,
+      "grad_norm": 2.3651468753814697,
+      "learning_rate": 2.8371242891957762e-06,
+      "loss": 1.8565,
+      "step": 1354
+    },
+    {
+      "epoch": 1.484931506849315,
+      "grad_norm": 2.3559513092041016,
+      "learning_rate": 2.835093419983753e-06,
+      "loss": 1.8765,
+      "step": 1355
+    },
+    {
+      "epoch": 1.486027397260274,
+      "grad_norm": 2.232041358947754,
+      "learning_rate": 2.833062550771731e-06,
+      "loss": 1.805,
+      "step": 1356
+    },
+    {
+      "epoch": 1.4871232876712328,
+      "grad_norm": 2.2423202991485596,
+      "learning_rate": 2.8310316815597076e-06,
+      "loss": 1.8137,
+      "step": 1357
+    },
+    {
+      "epoch": 1.4882191780821918,
+      "grad_norm": 2.3454222679138184,
+      "learning_rate": 2.8290008123476853e-06,
+      "loss": 1.7839,
+      "step": 1358
+    },
+    {
+      "epoch": 1.4893150684931507,
+      "grad_norm": 2.351266384124756,
+      "learning_rate": 2.826969943135662e-06,
+      "loss": 1.8208,
+      "step": 1359
+    },
+    {
+      "epoch": 1.4904109589041097,
+      "grad_norm": 2.3025388717651367,
+      "learning_rate": 2.8249390739236395e-06,
+      "loss": 1.8059,
+      "step": 1360
+    },
+    {
+      "epoch": 1.4904109589041097,
+      "eval_loss": 2.0235962867736816,
+      "eval_runtime": 37.1359,
+      "eval_samples_per_second": 26.928,
+      "eval_steps_per_second": 1.131,
+      "step": 1360
+    },
+    {
+      "epoch": 1.4915068493150685,
+      "grad_norm": 2.2277750968933105,
+      "learning_rate": 2.8229082047116167e-06,
+      "loss": 1.8238,
+      "step": 1361
+    },
+    {
+      "epoch": 1.4926027397260273,
+      "grad_norm": 2.318286180496216,
+      "learning_rate": 2.820877335499594e-06,
+      "loss": 1.7867,
+      "step": 1362
+    },
+    {
+      "epoch": 1.4936986301369863,
+      "grad_norm": 2.296259880065918,
+      "learning_rate": 2.8188464662875713e-06,
+      "loss": 1.8068,
+      "step": 1363
+    },
+    {
+      "epoch": 1.4947945205479451,
+      "grad_norm": 2.2617714405059814,
+      "learning_rate": 2.8168155970755486e-06,
+      "loss": 1.7607,
+      "step": 1364
+    },
+    {
+      "epoch": 1.4958904109589042,
+      "grad_norm": 2.318294048309326,
+      "learning_rate": 2.814784727863526e-06,
+      "loss": 1.7872,
+      "step": 1365
+    },
+    {
+      "epoch": 1.496986301369863,
+      "grad_norm": 2.260286808013916,
+      "learning_rate": 2.812753858651503e-06,
+      "loss": 1.8013,
+      "step": 1366
+    },
+    {
+      "epoch": 1.498082191780822,
+      "grad_norm": 2.222074508666992,
+      "learning_rate": 2.8107229894394804e-06,
+      "loss": 1.8338,
+      "step": 1367
+    },
+    {
+      "epoch": 1.4991780821917808,
+      "grad_norm": 2.6403002738952637,
+      "learning_rate": 2.8086921202274577e-06,
+      "loss": 1.8126,
+      "step": 1368
+    },
+    {
+      "epoch": 1.5002739726027396,
+      "grad_norm": 2.2439680099487305,
+      "learning_rate": 2.806661251015435e-06,
+      "loss": 1.7669,
+      "step": 1369
+    },
+    {
+      "epoch": 1.5013698630136987,
+      "grad_norm": 2.3914687633514404,
+      "learning_rate": 2.8046303818034122e-06,
+      "loss": 1.8421,
+      "step": 1370
+    },
+    {
+      "epoch": 1.5013698630136987,
+      "eval_loss": 2.0222702026367188,
+      "eval_runtime": 37.1598,
+      "eval_samples_per_second": 26.911,
+      "eval_steps_per_second": 1.13,
+      "step": 1370
+    },
+    {
+      "epoch": 1.5024657534246575,
+      "grad_norm": 2.3510093688964844,
+      "learning_rate": 2.802599512591389e-06,
+      "loss": 1.8902,
+      "step": 1371
+    },
+    {
+      "epoch": 1.5035616438356163,
+      "grad_norm": 2.4427363872528076,
+      "learning_rate": 2.8005686433793668e-06,
+      "loss": 1.9376,
+      "step": 1372
+    },
+    {
+      "epoch": 1.5046575342465753,
+      "grad_norm": 2.299356460571289,
+      "learning_rate": 2.7985377741673436e-06,
+      "loss": 1.812,
+      "step": 1373
+    },
+    {
+      "epoch": 1.5057534246575344,
+      "grad_norm": 2.3738906383514404,
+      "learning_rate": 2.7965069049553213e-06,
+      "loss": 1.7666,
+      "step": 1374
+    },
+    {
+      "epoch": 1.5068493150684932,
+      "grad_norm": 2.3620493412017822,
+      "learning_rate": 2.794476035743298e-06,
+      "loss": 1.824,
+      "step": 1375
+    },
+    {
+      "epoch": 1.507945205479452,
+      "grad_norm": 2.539048671722412,
+      "learning_rate": 2.792445166531276e-06,
+      "loss": 1.8712,
+      "step": 1376
+    },
+    {
+      "epoch": 1.509041095890411,
+      "grad_norm": 2.2931456565856934,
+      "learning_rate": 2.7904142973192527e-06,
+      "loss": 1.8414,
+      "step": 1377
+    },
+    {
+      "epoch": 1.5101369863013698,
+      "grad_norm": 2.4368224143981934,
+      "learning_rate": 2.7883834281072304e-06,
+      "loss": 1.8419,
+      "step": 1378
+    },
+    {
+      "epoch": 1.5112328767123286,
+      "grad_norm": 2.298302173614502,
+      "learning_rate": 2.7863525588952073e-06,
+      "loss": 1.7846,
+      "step": 1379
+    },
+    {
+      "epoch": 1.5123287671232877,
+      "grad_norm": 2.3726806640625,
+      "learning_rate": 2.784321689683185e-06,
+      "loss": 1.7733,
+      "step": 1380
+    },
+    {
+      "epoch": 1.5123287671232877,
+      "eval_loss": 2.0201807022094727,
+      "eval_runtime": 37.1012,
+      "eval_samples_per_second": 26.953,
+      "eval_steps_per_second": 1.132,
+      "step": 1380
+    },
+    {
+      "epoch": 1.5134246575342467,
+      "grad_norm": 2.6515002250671387,
+      "learning_rate": 2.782290820471162e-06,
+      "loss": 1.7764,
+      "step": 1381
+    },
+    {
+      "epoch": 1.5145205479452055,
+      "grad_norm": 2.273272752761841,
+      "learning_rate": 2.7802599512591395e-06,
+      "loss": 1.7747,
+      "step": 1382
+    },
+    {
+      "epoch": 1.5156164383561643,
+      "grad_norm": 2.570009708404541,
+      "learning_rate": 2.7782290820471164e-06,
+      "loss": 1.8612,
+      "step": 1383
+    },
+    {
+      "epoch": 1.5167123287671234,
+      "grad_norm": 2.567878007888794,
+      "learning_rate": 2.7761982128350932e-06,
+      "loss": 1.8402,
+      "step": 1384
+    },
+    {
+      "epoch": 1.5178082191780822,
+      "grad_norm": 2.3768470287323,
+      "learning_rate": 2.774167343623071e-06,
+      "loss": 1.8473,
+      "step": 1385
+    },
+    {
+      "epoch": 1.518904109589041,
+      "grad_norm": 2.1383397579193115,
+      "learning_rate": 2.772136474411048e-06,
+      "loss": 1.8593,
+      "step": 1386
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 2.4437034130096436,
+      "learning_rate": 2.7701056051990255e-06,
+      "loss": 1.792,
+      "step": 1387
+    },
+    {
+      "epoch": 1.521095890410959,
+      "grad_norm": 2.433933734893799,
+      "learning_rate": 2.7680747359870023e-06,
+      "loss": 1.7312,
+      "step": 1388
+    },
+    {
+      "epoch": 1.5221917808219179,
+      "grad_norm": 2.340994119644165,
+      "learning_rate": 2.76604386677498e-06,
+      "loss": 1.8326,
+      "step": 1389
+    },
+    {
+      "epoch": 1.5232876712328767,
+      "grad_norm": 2.4730165004730225,
+      "learning_rate": 2.764012997562957e-06,
+      "loss": 1.8129,
+      "step": 1390
+    },
+    {
+      "epoch": 1.5232876712328767,
+      "eval_loss": 2.0158324241638184,
+      "eval_runtime": 37.0997,
+      "eval_samples_per_second": 26.954,
+      "eval_steps_per_second": 1.132,
+      "step": 1390
+    },
+    {
+      "epoch": 1.5243835616438357,
+      "grad_norm": 2.2898151874542236,
+      "learning_rate": 2.7619821283509346e-06,
+      "loss": 1.8726,
+      "step": 1391
+    },
+    {
+      "epoch": 1.5254794520547945,
+      "grad_norm": 2.255439519882202,
+      "learning_rate": 2.7599512591389114e-06,
+      "loss": 1.7587,
+      "step": 1392
+    },
+    {
+      "epoch": 1.5265753424657533,
+      "grad_norm": 2.4269862174987793,
+      "learning_rate": 2.757920389926889e-06,
+      "loss": 1.8498,
+      "step": 1393
+    },
+    {
+      "epoch": 1.5276712328767124,
+      "grad_norm": 2.3411412239074707,
+      "learning_rate": 2.755889520714866e-06,
+      "loss": 1.8103,
+      "step": 1394
+    },
+    {
+      "epoch": 1.5287671232876714,
+      "grad_norm": 2.2859277725219727,
+      "learning_rate": 2.7538586515028437e-06,
+      "loss": 1.7909,
+      "step": 1395
+    },
+    {
+      "epoch": 1.52986301369863,
+      "grad_norm": 2.2675201892852783,
+      "learning_rate": 2.7518277822908205e-06,
+      "loss": 1.7672,
+      "step": 1396
+    },
+    {
+      "epoch": 1.530958904109589,
+      "grad_norm": 2.240610361099243,
+      "learning_rate": 2.749796913078798e-06,
+      "loss": 1.8162,
+      "step": 1397
+    },
+    {
+      "epoch": 1.532054794520548,
+      "grad_norm": 2.272045373916626,
+      "learning_rate": 2.747766043866775e-06,
+      "loss": 1.8593,
+      "step": 1398
+    },
+    {
+      "epoch": 1.5331506849315069,
+      "grad_norm": 2.1736183166503906,
+      "learning_rate": 2.7457351746547524e-06,
+      "loss": 1.8154,
+      "step": 1399
+    },
+    {
+      "epoch": 1.5342465753424657,
+      "grad_norm": 2.197085380554199,
+      "learning_rate": 2.7437043054427297e-06,
+      "loss": 1.8758,
+      "step": 1400
+    },
+    {
+      "epoch": 1.5342465753424657,
+      "eval_loss": 2.017298698425293,
+      "eval_runtime": 37.1666,
+      "eval_samples_per_second": 26.906,
+      "eval_steps_per_second": 1.13,
+      "step": 1400
+    },
+    {
+      "epoch": 1.5353424657534247,
+      "grad_norm": 2.2274084091186523,
+      "learning_rate": 2.741673436230707e-06,
+      "loss": 1.9241,
+      "step": 1401
+    },
+    {
+      "epoch": 1.5364383561643835,
+      "grad_norm": 2.3320305347442627,
+      "learning_rate": 2.739642567018684e-06,
+      "loss": 1.8209,
+      "step": 1402
+    },
+    {
+      "epoch": 1.5375342465753423,
+      "grad_norm": 2.370651960372925,
+      "learning_rate": 2.7376116978066615e-06,
+      "loss": 1.7897,
+      "step": 1403
+    },
+    {
+      "epoch": 1.5386301369863014,
+      "grad_norm": 2.387906312942505,
+      "learning_rate": 2.7355808285946388e-06,
+      "loss": 1.8385,
+      "step": 1404
+    },
+    {
+      "epoch": 1.5397260273972604,
+      "grad_norm": 2.340228796005249,
+      "learning_rate": 2.733549959382616e-06,
+      "loss": 1.8279,
+      "step": 1405
+    },
+    {
+      "epoch": 1.5408219178082192,
+      "grad_norm": 2.338785409927368,
+      "learning_rate": 2.7315190901705933e-06,
+      "loss": 1.7887,
+      "step": 1406
+    },
+    {
+      "epoch": 1.541917808219178,
+      "grad_norm": 2.350057601928711,
+      "learning_rate": 2.7294882209585706e-06,
+      "loss": 1.8772,
+      "step": 1407
+    },
+    {
+      "epoch": 1.543013698630137,
+      "grad_norm": 2.1129322052001953,
+      "learning_rate": 2.727457351746548e-06,
+      "loss": 1.8393,
+      "step": 1408
+    },
+    {
+      "epoch": 1.5441095890410959,
+      "grad_norm": 2.282553195953369,
+      "learning_rate": 2.725426482534525e-06,
+      "loss": 1.7893,
+      "step": 1409
+    },
+    {
+      "epoch": 1.5452054794520547,
+      "grad_norm": 2.3291871547698975,
+      "learning_rate": 2.723395613322502e-06,
+      "loss": 1.7921,
+      "step": 1410
+    },
+    {
+      "epoch": 1.5452054794520547,
+      "eval_loss": 2.0166046619415283,
+      "eval_runtime": 36.7906,
+      "eval_samples_per_second": 27.181,
+      "eval_steps_per_second": 1.142,
+      "step": 1410
+    },
+    {
+      "epoch": 1.5463013698630137,
+      "grad_norm": 2.3050696849823,
+      "learning_rate": 2.7213647441104797e-06,
+      "loss": 1.8292,
+      "step": 1411
+    },
+    {
+      "epoch": 1.5473972602739727,
+      "grad_norm": 2.2762458324432373,
+      "learning_rate": 2.7193338748984565e-06,
+      "loss": 1.811,
+      "step": 1412
+    },
+    {
+      "epoch": 1.5484931506849315,
+      "grad_norm": 2.5544228553771973,
+      "learning_rate": 2.7173030056864342e-06,
+      "loss": 1.8203,
+      "step": 1413
+    },
+    {
+      "epoch": 1.5495890410958904,
+      "grad_norm": 2.6392276287078857,
+      "learning_rate": 2.715272136474411e-06,
+      "loss": 1.9724,
+      "step": 1414
+    },
+    {
+      "epoch": 1.5506849315068494,
+      "grad_norm": 2.445366382598877,
+      "learning_rate": 2.713241267262389e-06,
+      "loss": 1.7589,
+      "step": 1415
+    },
+    {
+      "epoch": 1.5517808219178082,
+      "grad_norm": 2.5305662155151367,
+      "learning_rate": 2.7112103980503656e-06,
+      "loss": 1.8311,
+      "step": 1416
+    },
+    {
+      "epoch": 1.552876712328767,
+      "grad_norm": 2.405482769012451,
+      "learning_rate": 2.7091795288383433e-06,
+      "loss": 1.7893,
+      "step": 1417
+    },
+    {
+      "epoch": 1.553972602739726,
+      "grad_norm": 2.258361577987671,
+      "learning_rate": 2.70714865962632e-06,
+      "loss": 1.8489,
+      "step": 1418
+    },
+    {
+      "epoch": 1.555068493150685,
+      "grad_norm": 2.1798577308654785,
+      "learning_rate": 2.705117790414298e-06,
+      "loss": 1.7983,
+      "step": 1419
+    },
+    {
+      "epoch": 1.5561643835616439,
+      "grad_norm": 2.1985363960266113,
+      "learning_rate": 2.7030869212022747e-06,
+      "loss": 1.7934,
+      "step": 1420
+    },
+    {
+      "epoch": 1.5561643835616439,
+      "eval_loss": 2.0219907760620117,
+      "eval_runtime": 36.9416,
+      "eval_samples_per_second": 27.07,
+      "eval_steps_per_second": 1.137,
+      "step": 1420
+    },
+    {
+      "epoch": 1.5572602739726027,
+      "grad_norm": 2.273252248764038,
+      "learning_rate": 2.7010560519902516e-06,
+      "loss": 1.8571,
+      "step": 1421
+    },
+    {
+      "epoch": 1.5583561643835617,
+      "grad_norm": 2.22920298576355,
+      "learning_rate": 2.6990251827782293e-06,
+      "loss": 1.7785,
+      "step": 1422
+    },
+    {
+      "epoch": 1.5594520547945205,
+      "grad_norm": 2.1784098148345947,
+      "learning_rate": 2.696994313566206e-06,
+      "loss": 1.8053,
+      "step": 1423
+    },
+    {
+      "epoch": 1.5605479452054793,
+      "grad_norm": 2.223404884338379,
+      "learning_rate": 2.694963444354184e-06,
+      "loss": 1.7965,
+      "step": 1424
+    },
+    {
+      "epoch": 1.5616438356164384,
+      "grad_norm": 2.2004787921905518,
+      "learning_rate": 2.6929325751421607e-06,
+      "loss": 1.8851,
+      "step": 1425
+    },
+    {
+      "epoch": 1.5627397260273974,
+      "grad_norm": 2.202467441558838,
+      "learning_rate": 2.6909017059301384e-06,
+      "loss": 1.7884,
+      "step": 1426
+    },
+    {
+      "epoch": 1.563835616438356,
+      "grad_norm": 2.337430238723755,
+      "learning_rate": 2.6888708367181153e-06,
+      "loss": 1.8543,
+      "step": 1427
+    },
+    {
+      "epoch": 1.564931506849315,
+      "grad_norm": 2.322848081588745,
+      "learning_rate": 2.686839967506093e-06,
+      "loss": 1.7607,
+      "step": 1428
+    },
+    {
+      "epoch": 1.566027397260274,
+      "grad_norm": 2.277765989303589,
+      "learning_rate": 2.68480909829407e-06,
+      "loss": 1.8354,
+      "step": 1429
+    },
+    {
+      "epoch": 1.5671232876712329,
+      "grad_norm": 2.3229448795318604,
+      "learning_rate": 2.6827782290820475e-06,
+      "loss": 1.7277,
+      "step": 1430
+    },
+    {
+      "epoch": 1.5671232876712329,
+      "eval_loss": 2.0220985412597656,
+      "eval_runtime": 37.0054,
+      "eval_samples_per_second": 27.023,
+      "eval_steps_per_second": 1.135,
+      "step": 1430
+    },
+    {
+      "epoch": 1.5682191780821917,
+      "grad_norm": 2.49545955657959,
+      "learning_rate": 2.6807473598700244e-06,
+      "loss": 1.7351,
+      "step": 1431
+    },
+    {
+      "epoch": 1.5693150684931507,
+      "grad_norm": 2.530553102493286,
+      "learning_rate": 2.678716490658002e-06,
+      "loss": 1.8614,
+      "step": 1432
+    },
+    {
+      "epoch": 1.5704109589041098,
+      "grad_norm": 2.3352842330932617,
+      "learning_rate": 2.676685621445979e-06,
+      "loss": 1.8218,
+      "step": 1433
+    },
+    {
+      "epoch": 1.5715068493150683,
+      "grad_norm": 2.3533058166503906,
+      "learning_rate": 2.674654752233956e-06,
+      "loss": 1.8282,
+      "step": 1434
+    },
+    {
+      "epoch": 1.5726027397260274,
+      "grad_norm": 2.5509746074676514,
+      "learning_rate": 2.6726238830219335e-06,
+      "loss": 1.8333,
+      "step": 1435
+    },
+    {
+      "epoch": 1.5736986301369864,
+      "grad_norm": 2.185175895690918,
+      "learning_rate": 2.6705930138099107e-06,
+      "loss": 1.8299,
+      "step": 1436
+    },
+    {
+      "epoch": 1.5747945205479452,
+      "grad_norm": 2.2338390350341797,
+      "learning_rate": 2.668562144597888e-06,
+      "loss": 1.857,
+      "step": 1437
+    },
+    {
+      "epoch": 1.575890410958904,
+      "grad_norm": 2.4094536304473877,
+      "learning_rate": 2.6665312753858653e-06,
+      "loss": 1.7333,
+      "step": 1438
+    },
+    {
+      "epoch": 1.576986301369863,
+      "grad_norm": 2.4650957584381104,
+      "learning_rate": 2.6645004061738426e-06,
+      "loss": 1.8237,
+      "step": 1439
+    },
+    {
+      "epoch": 1.5780821917808219,
+      "grad_norm": 2.427415132522583,
+      "learning_rate": 2.66246953696182e-06,
+      "loss": 1.8206,
+      "step": 1440
+    },
+    {
+      "epoch": 1.5780821917808219,
+      "eval_loss": 2.0222699642181396,
+      "eval_runtime": 37.0252,
+      "eval_samples_per_second": 27.009,
+      "eval_steps_per_second": 1.134,
+      "step": 1440
+    },
+    {
+      "epoch": 1.5791780821917807,
+      "grad_norm": 2.3878679275512695,
+      "learning_rate": 2.660438667749797e-06,
+      "loss": 1.775,
+      "step": 1441
+    },
+    {
+      "epoch": 1.5802739726027397,
+      "grad_norm": 2.317819595336914,
+      "learning_rate": 2.6584077985377744e-06,
+      "loss": 1.7479,
+      "step": 1442
+    },
+    {
+      "epoch": 1.5813698630136988,
+      "grad_norm": 2.34951114654541,
+      "learning_rate": 2.6563769293257517e-06,
+      "loss": 1.8025,
+      "step": 1443
+    },
+    {
+      "epoch": 1.5824657534246576,
+      "grad_norm": 2.2849795818328857,
+      "learning_rate": 2.654346060113729e-06,
+      "loss": 1.8152,
+      "step": 1444
+    },
+    {
+      "epoch": 1.5835616438356164,
+      "grad_norm": 2.211489677429199,
+      "learning_rate": 2.6523151909017062e-06,
+      "loss": 1.7666,
+      "step": 1445
+    },
+    {
+      "epoch": 1.5846575342465754,
+      "grad_norm": 2.1876230239868164,
+      "learning_rate": 2.6502843216896835e-06,
+      "loss": 1.7933,
+      "step": 1446
+    },
+    {
+      "epoch": 1.5857534246575342,
+      "grad_norm": 2.367628335952759,
+      "learning_rate": 2.6482534524776603e-06,
+      "loss": 1.7756,
+      "step": 1447
+    },
+    {
+      "epoch": 1.586849315068493,
+      "grad_norm": 2.1198348999023438,
+      "learning_rate": 2.646222583265638e-06,
+      "loss": 1.8908,
+      "step": 1448
+    },
+    {
+      "epoch": 1.587945205479452,
+      "grad_norm": 2.192788600921631,
+      "learning_rate": 2.644191714053615e-06,
+      "loss": 1.8432,
+      "step": 1449
+    },
+    {
+      "epoch": 1.589041095890411,
+      "grad_norm": 2.4480655193328857,
+      "learning_rate": 2.6421608448415926e-06,
+      "loss": 1.8223,
+      "step": 1450
+    },
+    {
+      "epoch": 1.589041095890411,
+      "eval_loss": 2.019976854324341,
+      "eval_runtime": 37.0117,
+      "eval_samples_per_second": 27.018,
+      "eval_steps_per_second": 1.135,
+      "step": 1450
+    },
+    {
+      "epoch": 1.59013698630137,
+      "grad_norm": 2.573078155517578,
+      "learning_rate": 2.6401299756295694e-06,
+      "loss": 1.7881,
+      "step": 1451
+    },
+    {
+      "epoch": 1.5912328767123287,
+      "grad_norm": 2.2820355892181396,
+      "learning_rate": 2.638099106417547e-06,
+      "loss": 1.7703,
+      "step": 1452
+    },
+    {
+      "epoch": 1.5923287671232877,
+      "grad_norm": 2.526477575302124,
+      "learning_rate": 2.636068237205524e-06,
+      "loss": 1.8281,
+      "step": 1453
+    },
+    {
+      "epoch": 1.5934246575342466,
+      "grad_norm": 2.873647689819336,
+      "learning_rate": 2.6340373679935017e-06,
+      "loss": 1.7771,
+      "step": 1454
+    },
+    {
+      "epoch": 1.5945205479452054,
+      "grad_norm": 2.3782413005828857,
+      "learning_rate": 2.6320064987814785e-06,
+      "loss": 1.8864,
+      "step": 1455
+    },
+    {
+      "epoch": 1.5956164383561644,
+      "grad_norm": 2.3153343200683594,
+      "learning_rate": 2.6299756295694562e-06,
+      "loss": 1.734,
+      "step": 1456
+    },
+    {
+      "epoch": 1.5967123287671234,
+      "grad_norm": 2.631880283355713,
+      "learning_rate": 2.627944760357433e-06,
+      "loss": 1.8295,
+      "step": 1457
+    },
+    {
+      "epoch": 1.5978082191780822,
+      "grad_norm": 2.5827910900115967,
+      "learning_rate": 2.6259138911454104e-06,
+      "loss": 1.8738,
+      "step": 1458
+    },
+    {
+      "epoch": 1.598904109589041,
+      "grad_norm": 2.3519935607910156,
+      "learning_rate": 2.6238830219333877e-06,
+      "loss": 1.8832,
+      "step": 1459
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 2.5236215591430664,
+      "learning_rate": 2.621852152721365e-06,
+      "loss": 1.8595,
+      "step": 1460
+    },
+    {
+      "epoch": 1.6,
+      "eval_loss": 2.0170512199401855,
+      "eval_runtime": 37.0302,
+      "eval_samples_per_second": 27.005,
+      "eval_steps_per_second": 1.134,
+      "step": 1460
+    },
+    {
+      "epoch": 1.601095890410959,
+      "grad_norm": 2.725083112716675,
+      "learning_rate": 2.619821283509342e-06,
+      "loss": 1.7476,
+      "step": 1461
+    },
+    {
+      "epoch": 1.6021917808219177,
+      "grad_norm": 2.212644100189209,
+      "learning_rate": 2.6177904142973195e-06,
+      "loss": 1.8475,
+      "step": 1462
+    },
+    {
+      "epoch": 1.6032876712328767,
+      "grad_norm": 2.346808910369873,
+      "learning_rate": 2.6157595450852968e-06,
+      "loss": 1.7762,
+      "step": 1463
+    },
+    {
+      "epoch": 1.6043835616438358,
+      "grad_norm": 2.702331304550171,
+      "learning_rate": 2.613728675873274e-06,
+      "loss": 1.8296,
+      "step": 1464
+    },
+    {
+      "epoch": 1.6054794520547944,
+      "grad_norm": 2.4175899028778076,
+      "learning_rate": 2.6116978066612513e-06,
+      "loss": 1.8476,
+      "step": 1465
+    },
+    {
+      "epoch": 1.6065753424657534,
+      "grad_norm": 2.325457811355591,
+      "learning_rate": 2.6096669374492286e-06,
+      "loss": 1.8706,
+      "step": 1466
+    },
+    {
+      "epoch": 1.6076712328767124,
+      "grad_norm": 2.383842945098877,
+      "learning_rate": 2.607636068237206e-06,
+      "loss": 1.8246,
+      "step": 1467
+    },
+    {
+      "epoch": 1.6087671232876712,
+      "grad_norm": 2.289515256881714,
+      "learning_rate": 2.605605199025183e-06,
+      "loss": 1.7728,
+      "step": 1468
+    },
+    {
+      "epoch": 1.60986301369863,
+      "grad_norm": 2.5116689205169678,
+      "learning_rate": 2.6035743298131604e-06,
+      "loss": 1.8359,
+      "step": 1469
+    },
+    {
+      "epoch": 1.610958904109589,
+      "grad_norm": 2.279695749282837,
+      "learning_rate": 2.6015434606011377e-06,
+      "loss": 1.8155,
+      "step": 1470
+    },
+    {
+      "epoch": 1.610958904109589,
+      "eval_loss": 2.0142691135406494,
+      "eval_runtime": 37.0119,
+      "eval_samples_per_second": 27.018,
+      "eval_steps_per_second": 1.135,
+      "step": 1470
+    },
+    {
+      "epoch": 1.612054794520548,
+      "grad_norm": 2.4397690296173096,
+      "learning_rate": 2.5995125913891145e-06,
+      "loss": 1.8424,
+      "step": 1471
+    },
+    {
+      "epoch": 1.6131506849315067,
+      "grad_norm": 2.433577299118042,
+      "learning_rate": 2.5974817221770922e-06,
+      "loss": 1.8196,
+      "step": 1472
+    },
+    {
+      "epoch": 1.6142465753424657,
+      "grad_norm": 2.365459442138672,
+      "learning_rate": 2.595450852965069e-06,
+      "loss": 1.8124,
+      "step": 1473
+    },
+    {
+      "epoch": 1.6153424657534248,
+      "grad_norm": 2.253133535385132,
+      "learning_rate": 2.593419983753047e-06,
+      "loss": 1.8443,
+      "step": 1474
+    },
+    {
+      "epoch": 1.6164383561643836,
+      "grad_norm": 2.227625608444214,
+      "learning_rate": 2.5913891145410236e-06,
+      "loss": 1.8524,
+      "step": 1475
+    },
+    {
+      "epoch": 1.6175342465753424,
+      "grad_norm": 2.403702974319458,
+      "learning_rate": 2.5893582453290013e-06,
+      "loss": 1.7983,
+      "step": 1476
+    },
+    {
+      "epoch": 1.6186301369863014,
+      "grad_norm": 2.24507999420166,
+      "learning_rate": 2.587327376116978e-06,
+      "loss": 1.8563,
+      "step": 1477
+    },
+    {
+      "epoch": 1.6197260273972602,
+      "grad_norm": 2.284700393676758,
+      "learning_rate": 2.585296506904956e-06,
+      "loss": 1.7958,
+      "step": 1478
+    },
+    {
+      "epoch": 1.620821917808219,
+      "grad_norm": 2.288215398788452,
+      "learning_rate": 2.5832656376929327e-06,
+      "loss": 1.8021,
+      "step": 1479
+    },
+    {
+      "epoch": 1.621917808219178,
+      "grad_norm": 2.2312939167022705,
+      "learning_rate": 2.5812347684809104e-06,
+      "loss": 1.7856,
+      "step": 1480
+    },
+    {
+      "epoch": 1.621917808219178,
+      "eval_loss": 2.01674222946167,
+      "eval_runtime": 36.9943,
+      "eval_samples_per_second": 27.031,
+      "eval_steps_per_second": 1.135,
+      "step": 1480
+    },
+    {
+      "epoch": 1.6230136986301371,
+      "grad_norm": 2.319363832473755,
+      "learning_rate": 2.5792038992688873e-06,
+      "loss": 1.7263,
+      "step": 1481
+    },
+    {
+      "epoch": 1.624109589041096,
+      "grad_norm": 2.4342033863067627,
+      "learning_rate": 2.577173030056865e-06,
+      "loss": 1.897,
+      "step": 1482
+    },
+    {
+      "epoch": 1.6252054794520547,
+      "grad_norm": 2.3052282333374023,
+      "learning_rate": 2.575142160844842e-06,
+      "loss": 1.7165,
+      "step": 1483
+    },
+    {
+      "epoch": 1.6263013698630138,
+      "grad_norm": 2.2476744651794434,
+      "learning_rate": 2.5731112916328187e-06,
+      "loss": 1.8347,
+      "step": 1484
+    },
+    {
+      "epoch": 1.6273972602739726,
+      "grad_norm": 2.3128387928009033,
+      "learning_rate": 2.5710804224207964e-06,
+      "loss": 1.8403,
+      "step": 1485
+    },
+    {
+      "epoch": 1.6284931506849314,
+      "grad_norm": 2.3483974933624268,
+      "learning_rate": 2.5690495532087733e-06,
+      "loss": 1.781,
+      "step": 1486
+    },
+    {
+      "epoch": 1.6295890410958904,
+      "grad_norm": 2.328298568725586,
+      "learning_rate": 2.567018683996751e-06,
+      "loss": 1.7356,
+      "step": 1487
+    },
+    {
+      "epoch": 1.6306849315068495,
+      "grad_norm": 2.2756922245025635,
+      "learning_rate": 2.564987814784728e-06,
+      "loss": 1.8449,
+      "step": 1488
+    },
+    {
+      "epoch": 1.6317808219178083,
+      "grad_norm": 2.363874912261963,
+      "learning_rate": 2.5629569455727055e-06,
+      "loss": 1.7767,
+      "step": 1489
+    },
+    {
+      "epoch": 1.632876712328767,
+      "grad_norm": 2.2872767448425293,
+      "learning_rate": 2.5609260763606824e-06,
+      "loss": 1.8052,
+      "step": 1490
+    },
+    {
+      "epoch": 1.632876712328767,
+      "eval_loss": 2.0134243965148926,
+      "eval_runtime": 37.029,
+      "eval_samples_per_second": 27.006,
+      "eval_steps_per_second": 1.134,
+      "step": 1490
+    },
+    {
+      "epoch": 1.633972602739726,
+      "grad_norm": 2.354851007461548,
+      "learning_rate": 2.55889520714866e-06,
+      "loss": 1.7953,
+      "step": 1491
+    },
+    {
+      "epoch": 1.635068493150685,
+      "grad_norm": 2.4440531730651855,
+      "learning_rate": 2.556864337936637e-06,
+      "loss": 1.9063,
+      "step": 1492
+    },
+    {
+      "epoch": 1.6361643835616437,
+      "grad_norm": 2.296835422515869,
+      "learning_rate": 2.5548334687246146e-06,
+      "loss": 1.7961,
+      "step": 1493
+    },
+    {
+      "epoch": 1.6372602739726028,
+      "grad_norm": 2.7421159744262695,
+      "learning_rate": 2.5528025995125915e-06,
+      "loss": 1.809,
+      "step": 1494
+    },
+    {
+      "epoch": 1.6383561643835618,
+      "grad_norm": 2.819125175476074,
+      "learning_rate": 2.550771730300569e-06,
+      "loss": 1.853,
+      "step": 1495
+    },
+    {
+      "epoch": 1.6394520547945204,
+      "grad_norm": 2.2536680698394775,
+      "learning_rate": 2.548740861088546e-06,
+      "loss": 1.8297,
+      "step": 1496
+    },
+    {
+      "epoch": 1.6405479452054794,
+      "grad_norm": 2.7130825519561768,
+      "learning_rate": 2.5467099918765233e-06,
+      "loss": 1.843,
+      "step": 1497
+    },
+    {
+      "epoch": 1.6416438356164385,
+      "grad_norm": 3.0625038146972656,
+      "learning_rate": 2.5446791226645006e-06,
+      "loss": 1.7462,
+      "step": 1498
+    },
+    {
+      "epoch": 1.6427397260273973,
+      "grad_norm": 2.3095271587371826,
+      "learning_rate": 2.542648253452478e-06,
+      "loss": 1.7363,
+      "step": 1499
+    },
+    {
+      "epoch": 1.643835616438356,
+      "grad_norm": 2.536547899246216,
+      "learning_rate": 2.540617384240455e-06,
+      "loss": 1.8422,
+      "step": 1500
+    },
+    {
+      "epoch": 1.643835616438356,
+      "eval_loss": 2.0126097202301025,
+      "eval_runtime": 37.0469,
+      "eval_samples_per_second": 26.993,
+      "eval_steps_per_second": 1.134,
+      "step": 1500
+    },
+    {
+      "epoch": 1.644931506849315,
+      "grad_norm": 2.7454309463500977,
+      "learning_rate": 2.5385865150284324e-06,
+      "loss": 1.8183,
+      "step": 1501
+    },
+    {
+      "epoch": 1.6460273972602741,
+      "grad_norm": 2.3443901538848877,
+      "learning_rate": 2.5365556458164097e-06,
+      "loss": 1.7827,
+      "step": 1502
+    },
+    {
+      "epoch": 1.6471232876712327,
+      "grad_norm": 2.2412657737731934,
+      "learning_rate": 2.534524776604387e-06,
+      "loss": 1.797,
+      "step": 1503
+    },
+    {
+      "epoch": 1.6482191780821918,
+      "grad_norm": 2.6193699836730957,
+      "learning_rate": 2.5324939073923642e-06,
+      "loss": 1.8103,
+      "step": 1504
+    },
+    {
+      "epoch": 1.6493150684931508,
+      "grad_norm": 2.363919258117676,
+      "learning_rate": 2.5304630381803415e-06,
+      "loss": 1.6987,
+      "step": 1505
+    },
+    {
+      "epoch": 1.6504109589041096,
+      "grad_norm": 2.303164482116699,
+      "learning_rate": 2.5284321689683188e-06,
+      "loss": 1.8005,
+      "step": 1506
+    },
+    {
+      "epoch": 1.6515068493150684,
+      "grad_norm": 2.444174289703369,
+      "learning_rate": 2.526401299756296e-06,
+      "loss": 1.8009,
+      "step": 1507
+    },
+    {
+      "epoch": 1.6526027397260274,
+      "grad_norm": 2.418454647064209,
+      "learning_rate": 2.524370430544273e-06,
+      "loss": 1.782,
+      "step": 1508
+    },
+    {
+      "epoch": 1.6536986301369863,
+      "grad_norm": 2.48840069770813,
+      "learning_rate": 2.5223395613322506e-06,
+      "loss": 1.8143,
+      "step": 1509
+    },
+    {
+      "epoch": 1.654794520547945,
+      "grad_norm": 2.2814347743988037,
+      "learning_rate": 2.5203086921202274e-06,
+      "loss": 1.7687,
+      "step": 1510
+    },
+    {
+      "epoch": 1.654794520547945,
+      "eval_loss": 2.0148253440856934,
+      "eval_runtime": 36.7833,
+      "eval_samples_per_second": 27.186,
+      "eval_steps_per_second": 1.142,
+      "step": 1510
+    },
+    {
+      "epoch": 1.655890410958904,
+      "grad_norm": 2.228933572769165,
+      "learning_rate": 2.518277822908205e-06,
+      "loss": 1.8416,
+      "step": 1511
+    },
+    {
+      "epoch": 1.6569863013698631,
+      "grad_norm": 2.3696742057800293,
+      "learning_rate": 2.516246953696182e-06,
+      "loss": 1.7868,
+      "step": 1512
+    },
+    {
+      "epoch": 1.658082191780822,
+      "grad_norm": 2.216360092163086,
+      "learning_rate": 2.5142160844841597e-06,
+      "loss": 1.8312,
+      "step": 1513
+    },
+    {
+      "epoch": 1.6591780821917808,
+      "grad_norm": 2.454345226287842,
+      "learning_rate": 2.5121852152721365e-06,
+      "loss": 1.8309,
+      "step": 1514
+    },
+    {
+      "epoch": 1.6602739726027398,
+      "grad_norm": 2.2106776237487793,
+      "learning_rate": 2.5101543460601142e-06,
+      "loss": 1.8155,
+      "step": 1515
+    },
+    {
+      "epoch": 1.6613698630136986,
+      "grad_norm": 2.289006233215332,
+      "learning_rate": 2.508123476848091e-06,
+      "loss": 1.8162,
+      "step": 1516
+    },
+    {
+      "epoch": 1.6624657534246574,
+      "grad_norm": 2.297246217727661,
+      "learning_rate": 2.506092607636069e-06,
+      "loss": 1.7559,
+      "step": 1517
+    },
+    {
+      "epoch": 1.6635616438356164,
+      "grad_norm": 2.3909482955932617,
+      "learning_rate": 2.5040617384240457e-06,
+      "loss": 1.8389,
+      "step": 1518
+    },
+    {
+      "epoch": 1.6646575342465755,
+      "grad_norm": 2.398171901702881,
+      "learning_rate": 2.5020308692120234e-06,
+      "loss": 1.7785,
+      "step": 1519
+    },
+    {
+      "epoch": 1.6657534246575343,
+      "grad_norm": 2.377707004547119,
+      "learning_rate": 2.5e-06,
+      "loss": 1.768,
+      "step": 1520
+    },
+    {
+      "epoch": 1.6657534246575343,
+      "eval_loss": 2.012815475463867,
+      "eval_runtime": 36.9603,
+      "eval_samples_per_second": 27.056,
+      "eval_steps_per_second": 1.136,
+      "step": 1520
+    },
+    {
+      "epoch": 1.666849315068493,
+      "grad_norm": 2.336160898208618,
+      "learning_rate": 2.4979691307879775e-06,
+      "loss": 1.8653,
+      "step": 1521
+    },
+    {
+      "epoch": 1.6679452054794521,
+      "grad_norm": 2.288424253463745,
+      "learning_rate": 2.4959382615759548e-06,
+      "loss": 1.7798,
+      "step": 1522
+    },
+    {
+      "epoch": 1.669041095890411,
+      "grad_norm": 2.2599472999572754,
+      "learning_rate": 2.493907392363932e-06,
+      "loss": 1.7736,
+      "step": 1523
+    },
+    {
+      "epoch": 1.6701369863013698,
+      "grad_norm": 2.242276430130005,
+      "learning_rate": 2.4918765231519093e-06,
+      "loss": 1.8753,
+      "step": 1524
+    },
+    {
+      "epoch": 1.6712328767123288,
+      "grad_norm": 2.1897733211517334,
+      "learning_rate": 2.4898456539398866e-06,
+      "loss": 1.8353,
+      "step": 1525
+    },
+    {
+      "epoch": 1.6723287671232878,
+      "grad_norm": 2.254009246826172,
+      "learning_rate": 2.487814784727864e-06,
+      "loss": 1.7825,
+      "step": 1526
+    },
+    {
+      "epoch": 1.6734246575342466,
+      "grad_norm": 2.3704216480255127,
+      "learning_rate": 2.4857839155158407e-06,
+      "loss": 1.8742,
+      "step": 1527
+    },
+    {
+      "epoch": 1.6745205479452054,
+      "grad_norm": 2.1590118408203125,
+      "learning_rate": 2.483753046303818e-06,
+      "loss": 1.7974,
+      "step": 1528
+    },
+    {
+      "epoch": 1.6756164383561645,
+      "grad_norm": 2.2389416694641113,
+      "learning_rate": 2.4817221770917953e-06,
+      "loss": 1.8007,
+      "step": 1529
+    },
+    {
+      "epoch": 1.6767123287671233,
+      "grad_norm": 2.324131488800049,
+      "learning_rate": 2.4796913078797725e-06,
+      "loss": 1.8079,
+      "step": 1530
+    },
+    {
+      "epoch": 1.6767123287671233,
+      "eval_loss": 2.013923168182373,
+      "eval_runtime": 37.0828,
+      "eval_samples_per_second": 26.967,
+      "eval_steps_per_second": 1.133,
+      "step": 1530
+    },
+    {
+      "epoch": 1.677808219178082,
+      "grad_norm": 2.406456708908081,
+      "learning_rate": 2.47766043866775e-06,
+      "loss": 1.8356,
+      "step": 1531
+    },
+    {
+      "epoch": 1.6789041095890411,
+      "grad_norm": 2.289919137954712,
+      "learning_rate": 2.475629569455727e-06,
+      "loss": 1.8239,
+      "step": 1532
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 2.2617733478546143,
+      "learning_rate": 2.4735987002437044e-06,
+      "loss": 1.8025,
+      "step": 1533
+    },
+    {
+      "epoch": 1.6810958904109587,
+      "grad_norm": 2.133652925491333,
+      "learning_rate": 2.4715678310316816e-06,
+      "loss": 1.7743,
+      "step": 1534
+    },
+    {
+      "epoch": 1.6821917808219178,
+      "grad_norm": 2.215322256088257,
+      "learning_rate": 2.469536961819659e-06,
+      "loss": 1.7979,
+      "step": 1535
+    },
+    {
+      "epoch": 1.6832876712328768,
+      "grad_norm": 2.336021900177002,
+      "learning_rate": 2.467506092607636e-06,
+      "loss": 1.8027,
+      "step": 1536
+    },
+    {
+      "epoch": 1.6843835616438356,
+      "grad_norm": 2.3567142486572266,
+      "learning_rate": 2.4654752233956135e-06,
+      "loss": 1.7877,
+      "step": 1537
+    },
+    {
+      "epoch": 1.6854794520547944,
+      "grad_norm": 2.2642927169799805,
+      "learning_rate": 2.4634443541835907e-06,
+      "loss": 1.7695,
+      "step": 1538
+    },
+    {
+      "epoch": 1.6865753424657535,
+      "grad_norm": 2.353303909301758,
+      "learning_rate": 2.461413484971568e-06,
+      "loss": 1.867,
+      "step": 1539
+    },
+    {
+      "epoch": 1.6876712328767123,
+      "grad_norm": 2.252074956893921,
+      "learning_rate": 2.4593826157595453e-06,
+      "loss": 1.8009,
+      "step": 1540
+    },
+    {
+      "epoch": 1.6876712328767123,
+      "eval_loss": 2.0113608837127686,
+      "eval_runtime": 37.0875,
+      "eval_samples_per_second": 26.963,
+      "eval_steps_per_second": 1.132,
+      "step": 1540
+    },
+    {
+      "epoch": 1.688767123287671,
+      "grad_norm": 2.2303407192230225,
+      "learning_rate": 2.4573517465475226e-06,
+      "loss": 1.8881,
+      "step": 1541
+    },
+    {
+      "epoch": 1.6898630136986301,
+      "grad_norm": 2.3359081745147705,
+      "learning_rate": 2.4553208773355e-06,
+      "loss": 1.7388,
+      "step": 1542
+    },
+    {
+      "epoch": 1.6909589041095892,
+      "grad_norm": 2.3783583641052246,
+      "learning_rate": 2.453290008123477e-06,
+      "loss": 1.7993,
+      "step": 1543
+    },
+    {
+      "epoch": 1.692054794520548,
+      "grad_norm": 2.159264326095581,
+      "learning_rate": 2.4512591389114544e-06,
+      "loss": 1.81,
+      "step": 1544
+    },
+    {
+      "epoch": 1.6931506849315068,
+      "grad_norm": 2.2232203483581543,
+      "learning_rate": 2.4492282696994317e-06,
+      "loss": 1.8095,
+      "step": 1545
+    },
+    {
+      "epoch": 1.6942465753424658,
+      "grad_norm": 2.1755056381225586,
+      "learning_rate": 2.447197400487409e-06,
+      "loss": 1.8149,
+      "step": 1546
+    },
+    {
+      "epoch": 1.6953424657534246,
+      "grad_norm": 2.325871467590332,
+      "learning_rate": 2.4451665312753862e-06,
+      "loss": 1.8154,
+      "step": 1547
+    },
+    {
+      "epoch": 1.6964383561643834,
+      "grad_norm": 2.286848545074463,
+      "learning_rate": 2.4431356620633635e-06,
+      "loss": 1.8409,
+      "step": 1548
+    },
+    {
+      "epoch": 1.6975342465753425,
+      "grad_norm": 2.359510898590088,
+      "learning_rate": 2.4411047928513408e-06,
+      "loss": 1.843,
+      "step": 1549
+    },
+    {
+      "epoch": 1.6986301369863015,
+      "grad_norm": 2.474428176879883,
+      "learning_rate": 2.439073923639318e-06,
+      "loss": 1.8089,
+      "step": 1550
+    },
+    {
+      "epoch": 1.6986301369863015,
+      "eval_loss": 2.0077731609344482,
+      "eval_runtime": 37.1092,
+      "eval_samples_per_second": 26.947,
+      "eval_steps_per_second": 1.132,
+      "step": 1550
+    },
+    {
+      "epoch": 1.6997260273972603,
+      "grad_norm": 2.3662912845611572,
+      "learning_rate": 2.4370430544272953e-06,
+      "loss": 1.8176,
+      "step": 1551
+    },
+    {
+      "epoch": 1.7008219178082191,
+      "grad_norm": 2.448948860168457,
+      "learning_rate": 2.435012185215272e-06,
+      "loss": 1.7776,
+      "step": 1552
+    },
+    {
+      "epoch": 1.7019178082191782,
+      "grad_norm": 2.333739757537842,
+      "learning_rate": 2.4329813160032495e-06,
+      "loss": 1.7552,
+      "step": 1553
+    },
+    {
+      "epoch": 1.703013698630137,
+      "grad_norm": 2.170365571975708,
+      "learning_rate": 2.4309504467912267e-06,
+      "loss": 1.7759,
+      "step": 1554
+    },
+    {
+      "epoch": 1.7041095890410958,
+      "grad_norm": 2.2343010902404785,
+      "learning_rate": 2.428919577579204e-06,
+      "loss": 1.8073,
+      "step": 1555
+    },
+    {
+      "epoch": 1.7052054794520548,
+      "grad_norm": 2.2245287895202637,
+      "learning_rate": 2.4268887083671813e-06,
+      "loss": 1.7826,
+      "step": 1556
+    },
+    {
+      "epoch": 1.7063013698630138,
+      "grad_norm": 2.339942216873169,
+      "learning_rate": 2.4248578391551586e-06,
+      "loss": 1.8766,
+      "step": 1557
+    },
+    {
+      "epoch": 1.7073972602739727,
+      "grad_norm": 2.1585116386413574,
+      "learning_rate": 2.422826969943136e-06,
+      "loss": 1.7605,
+      "step": 1558
+    },
+    {
+      "epoch": 1.7084931506849315,
+      "grad_norm": 2.2108142375946045,
+      "learning_rate": 2.420796100731113e-06,
+      "loss": 1.7855,
+      "step": 1559
+    },
+    {
+      "epoch": 1.7095890410958905,
+      "grad_norm": 2.192457675933838,
+      "learning_rate": 2.4187652315190904e-06,
+      "loss": 1.7489,
+      "step": 1560
+    },
+    {
+      "epoch": 1.7095890410958905,
+      "eval_loss": 2.0085604190826416,
+      "eval_runtime": 37.1105,
+      "eval_samples_per_second": 26.947,
+      "eval_steps_per_second": 1.132,
+      "step": 1560
+    },
+    {
+      "epoch": 1.7106849315068493,
+      "grad_norm": 2.291106939315796,
+      "learning_rate": 2.4167343623070677e-06,
+      "loss": 1.7981,
+      "step": 1561
+    },
+    {
+      "epoch": 1.7117808219178081,
+      "grad_norm": 2.509237766265869,
+      "learning_rate": 2.414703493095045e-06,
+      "loss": 1.8146,
+      "step": 1562
+    },
+    {
+      "epoch": 1.7128767123287671,
+      "grad_norm": 2.497162342071533,
+      "learning_rate": 2.4126726238830222e-06,
+      "loss": 1.807,
+      "step": 1563
+    },
+    {
+      "epoch": 1.7139726027397262,
+      "grad_norm": 2.256833076477051,
+      "learning_rate": 2.410641754670999e-06,
+      "loss": 1.8304,
+      "step": 1564
+    },
+    {
+      "epoch": 1.7150684931506848,
+      "grad_norm": 2.2523772716522217,
+      "learning_rate": 2.4086108854589763e-06,
+      "loss": 1.7794,
+      "step": 1565
+    },
+    {
+      "epoch": 1.7161643835616438,
+      "grad_norm": 2.289182424545288,
+      "learning_rate": 2.4065800162469536e-06,
+      "loss": 1.8425,
+      "step": 1566
+    },
+    {
+      "epoch": 1.7172602739726028,
+      "grad_norm": 2.734487771987915,
+      "learning_rate": 2.404549147034931e-06,
+      "loss": 1.7791,
+      "step": 1567
+    },
+    {
+      "epoch": 1.7183561643835616,
+      "grad_norm": 2.3307888507843018,
+      "learning_rate": 2.402518277822908e-06,
+      "loss": 1.8064,
+      "step": 1568
+    },
+    {
+      "epoch": 1.7194520547945205,
+      "grad_norm": 2.5016298294067383,
+      "learning_rate": 2.4004874086108854e-06,
+      "loss": 1.8544,
+      "step": 1569
+    },
+    {
+      "epoch": 1.7205479452054795,
+      "grad_norm": 2.142246961593628,
+      "learning_rate": 2.3984565393988627e-06,
+      "loss": 1.7653,
+      "step": 1570
+    },
+    {
+      "epoch": 1.7205479452054795,
+      "eval_loss": 2.0068864822387695,
+      "eval_runtime": 37.1078,
+      "eval_samples_per_second": 26.949,
+      "eval_steps_per_second": 1.132,
+      "step": 1570
+    },
+    {
+      "epoch": 1.7216438356164385,
+      "grad_norm": 2.5692505836486816,
+      "learning_rate": 2.39642567018684e-06,
+      "loss": 1.8444,
+      "step": 1571
+    },
+    {
+      "epoch": 1.7227397260273971,
+      "grad_norm": 2.2596213817596436,
+      "learning_rate": 2.3943948009748173e-06,
+      "loss": 1.7694,
+      "step": 1572
+    },
+    {
+      "epoch": 1.7238356164383561,
+      "grad_norm": 2.230984926223755,
+      "learning_rate": 2.3923639317627945e-06,
+      "loss": 1.786,
+      "step": 1573
+    },
+    {
+      "epoch": 1.7249315068493152,
+      "grad_norm": 2.366178274154663,
+      "learning_rate": 2.390333062550772e-06,
+      "loss": 1.8138,
+      "step": 1574
+    },
+    {
+      "epoch": 1.726027397260274,
+      "grad_norm": 2.23783016204834,
+      "learning_rate": 2.388302193338749e-06,
+      "loss": 1.7615,
+      "step": 1575
+    },
+    {
+      "epoch": 1.7271232876712328,
+      "grad_norm": 2.3766677379608154,
+      "learning_rate": 2.3862713241267264e-06,
+      "loss": 1.6811,
+      "step": 1576
+    },
+    {
+      "epoch": 1.7282191780821918,
+      "grad_norm": 2.3434040546417236,
+      "learning_rate": 2.3842404549147037e-06,
+      "loss": 1.8806,
+      "step": 1577
+    },
+    {
+      "epoch": 1.7293150684931506,
+      "grad_norm": 2.2444276809692383,
+      "learning_rate": 2.382209585702681e-06,
+      "loss": 1.7968,
+      "step": 1578
+    },
+    {
+      "epoch": 1.7304109589041095,
+      "grad_norm": 2.2524919509887695,
+      "learning_rate": 2.380178716490658e-06,
+      "loss": 1.7602,
+      "step": 1579
+    },
+    {
+      "epoch": 1.7315068493150685,
+      "grad_norm": 2.3310587406158447,
+      "learning_rate": 2.3781478472786355e-06,
+      "loss": 1.8414,
+      "step": 1580
+    },
+    {
+      "epoch": 1.7315068493150685,
+      "eval_loss": 1.9981482028961182,
+      "eval_runtime": 37.0827,
+      "eval_samples_per_second": 26.967,
+      "eval_steps_per_second": 1.133,
+      "step": 1580
+    },
+    {
+      "epoch": 1.7326027397260275,
+      "grad_norm": 2.4081802368164062,
+      "learning_rate": 2.3761169780666128e-06,
+      "loss": 1.8002,
+      "step": 1581
+    },
+    {
+      "epoch": 1.7336986301369863,
+      "grad_norm": 2.293506622314453,
+      "learning_rate": 2.37408610885459e-06,
+      "loss": 1.8365,
+      "step": 1582
+    },
+    {
+      "epoch": 1.7347945205479451,
+      "grad_norm": 2.2015790939331055,
+      "learning_rate": 2.3720552396425673e-06,
+      "loss": 1.8571,
+      "step": 1583
+    },
+    {
+      "epoch": 1.7358904109589042,
+      "grad_norm": 2.308809518814087,
+      "learning_rate": 2.3700243704305446e-06,
+      "loss": 1.7821,
+      "step": 1584
+    },
+    {
+      "epoch": 1.736986301369863,
+      "grad_norm": 2.376842975616455,
+      "learning_rate": 2.367993501218522e-06,
+      "loss": 1.8701,
+      "step": 1585
+    },
+    {
+      "epoch": 1.7380821917808218,
+      "grad_norm": 2.4102022647857666,
+      "learning_rate": 2.365962632006499e-06,
+      "loss": 1.8055,
+      "step": 1586
+    },
+    {
+      "epoch": 1.7391780821917808,
+      "grad_norm": 2.2734079360961914,
+      "learning_rate": 2.3639317627944764e-06,
+      "loss": 1.8117,
+      "step": 1587
+    },
+    {
+      "epoch": 1.7402739726027399,
+      "grad_norm": 2.3426599502563477,
+      "learning_rate": 2.3619008935824537e-06,
+      "loss": 1.8337,
+      "step": 1588
+    },
+    {
+      "epoch": 1.7413698630136987,
+      "grad_norm": 2.340383768081665,
+      "learning_rate": 2.3598700243704305e-06,
+      "loss": 1.8247,
+      "step": 1589
+    },
+    {
+      "epoch": 1.7424657534246575,
+      "grad_norm": 2.2561380863189697,
+      "learning_rate": 2.357839155158408e-06,
+      "loss": 1.8249,
+      "step": 1590
+    },
+    {
+      "epoch": 1.7424657534246575,
+      "eval_loss": 1.9979513883590698,
+      "eval_runtime": 37.0948,
+      "eval_samples_per_second": 26.958,
+      "eval_steps_per_second": 1.132,
+      "step": 1590
+    },
+    {
+      "epoch": 1.7435616438356165,
+      "grad_norm": 2.2459654808044434,
+      "learning_rate": 2.355808285946385e-06,
+      "loss": 1.8273,
+      "step": 1591
+    },
+    {
+      "epoch": 1.7446575342465753,
+      "grad_norm": 2.2017974853515625,
+      "learning_rate": 2.3537774167343624e-06,
+      "loss": 1.828,
+      "step": 1592
+    },
+    {
+      "epoch": 1.7457534246575341,
+      "grad_norm": 2.286304235458374,
+      "learning_rate": 2.3517465475223396e-06,
+      "loss": 1.8497,
+      "step": 1593
+    },
+    {
+      "epoch": 1.7468493150684932,
+      "grad_norm": 2.296494722366333,
+      "learning_rate": 2.349715678310317e-06,
+      "loss": 1.8155,
+      "step": 1594
+    },
+    {
+      "epoch": 1.7479452054794522,
+      "grad_norm": 2.2989203929901123,
+      "learning_rate": 2.347684809098294e-06,
+      "loss": 1.8214,
+      "step": 1595
+    },
+    {
+      "epoch": 1.749041095890411,
+      "grad_norm": 2.3034965991973877,
+      "learning_rate": 2.3456539398862715e-06,
+      "loss": 1.7792,
+      "step": 1596
+    },
+    {
+      "epoch": 1.7501369863013698,
+      "grad_norm": 2.3823719024658203,
+      "learning_rate": 2.3436230706742487e-06,
+      "loss": 1.8326,
+      "step": 1597
+    },
+    {
+      "epoch": 1.7512328767123289,
+      "grad_norm": 2.231095552444458,
+      "learning_rate": 2.341592201462226e-06,
+      "loss": 1.7713,
+      "step": 1598
+    },
+    {
+      "epoch": 1.7523287671232877,
+      "grad_norm": 2.287524938583374,
+      "learning_rate": 2.3395613322502033e-06,
+      "loss": 1.8143,
+      "step": 1599
+    },
+    {
+      "epoch": 1.7534246575342465,
+      "grad_norm": 2.267552614212036,
+      "learning_rate": 2.3375304630381806e-06,
+      "loss": 1.7883,
+      "step": 1600
+    },
+    {
+      "epoch": 1.7534246575342465,
+      "eval_loss": 1.998821496963501,
+      "eval_runtime": 37.1222,
+      "eval_samples_per_second": 26.938,
+      "eval_steps_per_second": 1.131,
+      "step": 1600
+    },
+    {
+      "epoch": 1.7545205479452055,
+      "grad_norm": 2.2986714839935303,
+      "learning_rate": 2.335499593826158e-06,
+      "loss": 1.7615,
+      "step": 1601
+    },
+    {
+      "epoch": 1.7556164383561645,
+      "grad_norm": 2.2913246154785156,
+      "learning_rate": 2.333468724614135e-06,
+      "loss": 1.7584,
+      "step": 1602
+    },
+    {
+      "epoch": 1.7567123287671231,
+      "grad_norm": 2.2486486434936523,
+      "learning_rate": 2.3314378554021124e-06,
+      "loss": 1.8089,
+      "step": 1603
+    },
+    {
+      "epoch": 1.7578082191780822,
+      "grad_norm": 2.319581985473633,
+      "learning_rate": 2.3294069861900897e-06,
+      "loss": 1.8364,
+      "step": 1604
+    },
+    {
+      "epoch": 1.7589041095890412,
+      "grad_norm": 2.2775609493255615,
+      "learning_rate": 2.327376116978067e-06,
+      "loss": 1.7661,
+      "step": 1605
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 2.3445472717285156,
+      "learning_rate": 2.3253452477660442e-06,
+      "loss": 1.7583,
+      "step": 1606
+    },
+    {
+      "epoch": 1.7610958904109588,
+      "grad_norm": 2.420865058898926,
+      "learning_rate": 2.3233143785540215e-06,
+      "loss": 1.8411,
+      "step": 1607
+    },
+    {
+      "epoch": 1.7621917808219179,
+      "grad_norm": 2.294821262359619,
+      "learning_rate": 2.3212835093419988e-06,
+      "loss": 1.7812,
+      "step": 1608
+    },
+    {
+      "epoch": 1.7632876712328767,
+      "grad_norm": 2.332784414291382,
+      "learning_rate": 2.319252640129976e-06,
+      "loss": 1.8252,
+      "step": 1609
+    },
+    {
+      "epoch": 1.7643835616438355,
+      "grad_norm": 2.433619499206543,
+      "learning_rate": 2.3172217709179533e-06,
+      "loss": 1.7709,
+      "step": 1610
+    },
+    {
+      "epoch": 1.7643835616438355,
+      "eval_loss": 2.0010287761688232,
+      "eval_runtime": 36.7858,
+      "eval_samples_per_second": 27.184,
+      "eval_steps_per_second": 1.142,
+      "step": 1610
+    },
+    {
+      "epoch": 1.7654794520547945,
+      "grad_norm": 2.532618522644043,
+      "learning_rate": 2.3151909017059306e-06,
+      "loss": 1.8677,
+      "step": 1611
+    },
+    {
+      "epoch": 1.7665753424657535,
+      "grad_norm": 2.354578733444214,
+      "learning_rate": 2.313160032493908e-06,
+      "loss": 1.8412,
+      "step": 1612
+    },
+    {
+      "epoch": 1.7676712328767124,
+      "grad_norm": 2.382774591445923,
+      "learning_rate": 2.311129163281885e-06,
+      "loss": 1.7543,
+      "step": 1613
+    },
+    {
+      "epoch": 1.7687671232876712,
+      "grad_norm": 2.3443398475646973,
+      "learning_rate": 2.309098294069862e-06,
+      "loss": 1.8267,
+      "step": 1614
+    },
+    {
+      "epoch": 1.7698630136986302,
+      "grad_norm": 2.3804502487182617,
+      "learning_rate": 2.3070674248578393e-06,
+      "loss": 1.7582,
+      "step": 1615
+    },
+    {
+      "epoch": 1.770958904109589,
+      "grad_norm": 2.3163580894470215,
+      "learning_rate": 2.3050365556458166e-06,
+      "loss": 1.7539,
+      "step": 1616
+    },
+    {
+      "epoch": 1.7720547945205478,
+      "grad_norm": 2.605738639831543,
+      "learning_rate": 2.303005686433794e-06,
+      "loss": 1.8124,
+      "step": 1617
+    },
+    {
+      "epoch": 1.7731506849315068,
+      "grad_norm": 2.3859915733337402,
+      "learning_rate": 2.300974817221771e-06,
+      "loss": 1.8271,
+      "step": 1618
+    },
+    {
+      "epoch": 1.7742465753424659,
+      "grad_norm": 2.3715479373931885,
+      "learning_rate": 2.2989439480097484e-06,
+      "loss": 1.7823,
+      "step": 1619
+    },
+    {
+      "epoch": 1.7753424657534247,
+      "grad_norm": 2.441100597381592,
+      "learning_rate": 2.2969130787977257e-06,
+      "loss": 1.7461,
+      "step": 1620
+    },
+    {
+      "epoch": 1.7753424657534247,
+      "eval_loss": 2.002152442932129,
+      "eval_runtime": 37.0246,
+      "eval_samples_per_second": 27.009,
+      "eval_steps_per_second": 1.134,
+      "step": 1620
+    },
+    {
+      "epoch": 1.7764383561643835,
+      "grad_norm": 2.4154021739959717,
+      "learning_rate": 2.294882209585703e-06,
+      "loss": 1.8012,
+      "step": 1621
+    },
+    {
+      "epoch": 1.7775342465753425,
+      "grad_norm": 2.333400249481201,
+      "learning_rate": 2.2928513403736802e-06,
+      "loss": 1.8222,
+      "step": 1622
+    },
+    {
+      "epoch": 1.7786301369863013,
+      "grad_norm": 2.3183937072753906,
+      "learning_rate": 2.2908204711616575e-06,
+      "loss": 1.7561,
+      "step": 1623
+    },
+    {
+      "epoch": 1.7797260273972602,
+      "grad_norm": 2.302823066711426,
+      "learning_rate": 2.2887896019496348e-06,
+      "loss": 1.7975,
+      "step": 1624
+    },
+    {
+      "epoch": 1.7808219178082192,
+      "grad_norm": 2.3702917098999023,
+      "learning_rate": 2.286758732737612e-06,
+      "loss": 1.8478,
+      "step": 1625
+    },
+    {
+      "epoch": 1.7819178082191782,
+      "grad_norm": 2.4997549057006836,
+      "learning_rate": 2.284727863525589e-06,
+      "loss": 1.7855,
+      "step": 1626
+    },
+    {
+      "epoch": 1.783013698630137,
+      "grad_norm": 2.296191930770874,
+      "learning_rate": 2.282696994313566e-06,
+      "loss": 1.7922,
+      "step": 1627
+    },
+    {
+      "epoch": 1.7841095890410958,
+      "grad_norm": 2.2567873001098633,
+      "learning_rate": 2.2806661251015434e-06,
+      "loss": 1.8455,
+      "step": 1628
+    },
+    {
+      "epoch": 1.7852054794520549,
+      "grad_norm": 2.393336534500122,
+      "learning_rate": 2.2786352558895207e-06,
+      "loss": 1.8141,
+      "step": 1629
+    },
+    {
+      "epoch": 1.7863013698630137,
+      "grad_norm": 2.610466480255127,
+      "learning_rate": 2.276604386677498e-06,
+      "loss": 1.7875,
+      "step": 1630
+    },
+    {
+      "epoch": 1.7863013698630137,
+      "eval_loss": 1.9985637664794922,
+      "eval_runtime": 37.1001,
+      "eval_samples_per_second": 26.954,
+      "eval_steps_per_second": 1.132,
+      "step": 1630
+    },
+    {
+      "epoch": 1.7873972602739725,
+      "grad_norm": 2.2163283824920654,
+      "learning_rate": 2.2745735174654753e-06,
+      "loss": 1.7479,
+      "step": 1631
+    },
+    {
+      "epoch": 1.7884931506849315,
+      "grad_norm": 2.2641165256500244,
+      "learning_rate": 2.2725426482534526e-06,
+      "loss": 1.7749,
+      "step": 1632
+    },
+    {
+      "epoch": 1.7895890410958906,
+      "grad_norm": 2.2786946296691895,
+      "learning_rate": 2.27051177904143e-06,
+      "loss": 1.8403,
+      "step": 1633
+    },
+    {
+      "epoch": 1.7906849315068492,
+      "grad_norm": 2.660356044769287,
+      "learning_rate": 2.268480909829407e-06,
+      "loss": 1.7994,
+      "step": 1634
+    },
+    {
+      "epoch": 1.7917808219178082,
+      "grad_norm": 2.322181224822998,
+      "learning_rate": 2.2664500406173844e-06,
+      "loss": 1.7917,
+      "step": 1635
+    },
+    {
+      "epoch": 1.7928767123287672,
+      "grad_norm": 2.3020682334899902,
+      "learning_rate": 2.2644191714053617e-06,
+      "loss": 1.7983,
+      "step": 1636
+    },
+    {
+      "epoch": 1.793972602739726,
+      "grad_norm": 2.5280368328094482,
+      "learning_rate": 2.262388302193339e-06,
+      "loss": 1.8378,
+      "step": 1637
+    },
+    {
+      "epoch": 1.7950684931506848,
+      "grad_norm": 2.3975653648376465,
+      "learning_rate": 2.260357432981316e-06,
+      "loss": 1.8081,
+      "step": 1638
+    },
+    {
+      "epoch": 1.7961643835616439,
+      "grad_norm": 2.419806957244873,
+      "learning_rate": 2.2583265637692935e-06,
+      "loss": 1.793,
+      "step": 1639
+    },
+    {
+      "epoch": 1.7972602739726027,
+      "grad_norm": 2.4601471424102783,
+      "learning_rate": 2.2562956945572708e-06,
+      "loss": 1.8116,
+      "step": 1640
+    },
+    {
+      "epoch": 1.7972602739726027,
+      "eval_loss": 1.9951634407043457,
+      "eval_runtime": 37.1095,
+      "eval_samples_per_second": 26.947,
+      "eval_steps_per_second": 1.132,
+      "step": 1640
+    },
+    {
+      "epoch": 1.7983561643835615,
+      "grad_norm": 2.17830753326416,
+      "learning_rate": 2.254264825345248e-06,
+      "loss": 1.7845,
+      "step": 1641
+    },
+    {
+      "epoch": 1.7994520547945205,
+      "grad_norm": 2.3332247734069824,
+      "learning_rate": 2.2522339561332253e-06,
+      "loss": 1.7945,
+      "step": 1642
+    },
+    {
+      "epoch": 1.8005479452054796,
+      "grad_norm": 2.3841960430145264,
+      "learning_rate": 2.2502030869212026e-06,
+      "loss": 1.7994,
+      "step": 1643
+    },
+    {
+      "epoch": 1.8016438356164384,
+      "grad_norm": 2.3669073581695557,
+      "learning_rate": 2.24817221770918e-06,
+      "loss": 1.8053,
+      "step": 1644
+    },
+    {
+      "epoch": 1.8027397260273972,
+      "grad_norm": 2.406858205795288,
+      "learning_rate": 2.246141348497157e-06,
+      "loss": 1.8647,
+      "step": 1645
+    },
+    {
+      "epoch": 1.8038356164383562,
+      "grad_norm": 2.2854161262512207,
+      "learning_rate": 2.2441104792851344e-06,
+      "loss": 1.8312,
+      "step": 1646
+    },
+    {
+      "epoch": 1.804931506849315,
+      "grad_norm": 2.252976655960083,
+      "learning_rate": 2.2420796100731117e-06,
+      "loss": 1.7834,
+      "step": 1647
+    },
+    {
+      "epoch": 1.8060273972602738,
+      "grad_norm": 2.4131624698638916,
+      "learning_rate": 2.240048740861089e-06,
+      "loss": 1.8204,
+      "step": 1648
+    },
+    {
+      "epoch": 1.8071232876712329,
+      "grad_norm": 2.4727940559387207,
+      "learning_rate": 2.2380178716490662e-06,
+      "loss": 1.8326,
+      "step": 1649
+    },
+    {
+      "epoch": 1.808219178082192,
+      "grad_norm": 2.4432709217071533,
+      "learning_rate": 2.2359870024370435e-06,
+      "loss": 1.6911,
+      "step": 1650
+    },
+    {
+      "epoch": 1.808219178082192,
+      "eval_loss": 1.996634840965271,
+      "eval_runtime": 37.072,
+      "eval_samples_per_second": 26.975,
+      "eval_steps_per_second": 1.133,
+      "step": 1650
+    },
+    {
+      "epoch": 1.8093150684931507,
+      "grad_norm": 2.3810362815856934,
+      "learning_rate": 2.2339561332250204e-06,
+      "loss": 1.8256,
+      "step": 1651
+    },
+    {
+      "epoch": 1.8104109589041095,
+      "grad_norm": 2.4489142894744873,
+      "learning_rate": 2.2319252640129976e-06,
+      "loss": 1.8368,
+      "step": 1652
+    },
+    {
+      "epoch": 1.8115068493150686,
+      "grad_norm": 2.458686113357544,
+      "learning_rate": 2.229894394800975e-06,
+      "loss": 1.8176,
+      "step": 1653
+    },
+    {
+      "epoch": 1.8126027397260274,
+      "grad_norm": 2.3977842330932617,
+      "learning_rate": 2.227863525588952e-06,
+      "loss": 1.7516,
+      "step": 1654
+    },
+    {
+      "epoch": 1.8136986301369862,
+      "grad_norm": 2.3777663707733154,
+      "learning_rate": 2.2258326563769295e-06,
+      "loss": 1.732,
+      "step": 1655
+    },
+    {
+      "epoch": 1.8147945205479452,
+      "grad_norm": 2.2874817848205566,
+      "learning_rate": 2.2238017871649067e-06,
+      "loss": 1.749,
+      "step": 1656
+    },
+    {
+      "epoch": 1.8158904109589042,
+      "grad_norm": 2.288114547729492,
+      "learning_rate": 2.221770917952884e-06,
+      "loss": 1.7357,
+      "step": 1657
+    },
+    {
+      "epoch": 1.816986301369863,
+      "grad_norm": 2.3532187938690186,
+      "learning_rate": 2.2197400487408613e-06,
+      "loss": 1.7953,
+      "step": 1658
+    },
+    {
+      "epoch": 1.8180821917808219,
+      "grad_norm": 2.2829396724700928,
+      "learning_rate": 2.2177091795288386e-06,
+      "loss": 1.8358,
+      "step": 1659
+    },
+    {
+      "epoch": 1.819178082191781,
+      "grad_norm": 2.3282570838928223,
+      "learning_rate": 2.215678310316816e-06,
+      "loss": 1.7448,
+      "step": 1660
+    },
+    {
+      "epoch": 1.819178082191781,
+      "eval_loss": 1.999374270439148,
+      "eval_runtime": 37.0739,
+      "eval_samples_per_second": 26.973,
+      "eval_steps_per_second": 1.133,
+      "step": 1660
+    },
+    {
+      "epoch": 1.8202739726027397,
+      "grad_norm": 2.474106788635254,
+      "learning_rate": 2.213647441104793e-06,
+      "loss": 1.7979,
+      "step": 1661
+    },
+    {
+      "epoch": 1.8213698630136985,
+      "grad_norm": 2.159498453140259,
+      "learning_rate": 2.2116165718927704e-06,
+      "loss": 1.8075,
+      "step": 1662
+    },
+    {
+      "epoch": 1.8224657534246576,
+      "grad_norm": 2.2503573894500732,
+      "learning_rate": 2.2095857026807473e-06,
+      "loss": 1.7742,
+      "step": 1663
+    },
+    {
+      "epoch": 1.8235616438356166,
+      "grad_norm": 2.4465060234069824,
+      "learning_rate": 2.2075548334687245e-06,
+      "loss": 1.8067,
+      "step": 1664
+    },
+    {
+      "epoch": 1.8246575342465754,
+      "grad_norm": 2.462158679962158,
+      "learning_rate": 2.205523964256702e-06,
+      "loss": 1.7456,
+      "step": 1665
+    },
+    {
+      "epoch": 1.8257534246575342,
+      "grad_norm": 2.354051113128662,
+      "learning_rate": 2.203493095044679e-06,
+      "loss": 1.7688,
+      "step": 1666
+    },
+    {
+      "epoch": 1.8268493150684932,
+      "grad_norm": 2.3165128231048584,
+      "learning_rate": 2.2014622258326564e-06,
+      "loss": 1.7986,
+      "step": 1667
+    },
+    {
+      "epoch": 1.827945205479452,
+      "grad_norm": 2.3131113052368164,
+      "learning_rate": 2.1994313566206336e-06,
+      "loss": 1.8348,
+      "step": 1668
+    },
+    {
+      "epoch": 1.8290410958904109,
+      "grad_norm": 2.303086996078491,
+      "learning_rate": 2.197400487408611e-06,
+      "loss": 1.8413,
+      "step": 1669
+    },
+    {
+      "epoch": 1.83013698630137,
+      "grad_norm": 2.392995834350586,
+      "learning_rate": 2.195369618196588e-06,
+      "loss": 1.7663,
+      "step": 1670
+    },
+    {
+      "epoch": 1.83013698630137,
+      "eval_loss": 1.9936721324920654,
+      "eval_runtime": 37.0717,
+      "eval_samples_per_second": 26.975,
+      "eval_steps_per_second": 1.133,
+      "step": 1670
+    },
+    {
+      "epoch": 1.831232876712329,
+      "grad_norm": 2.323596477508545,
+      "learning_rate": 2.1933387489845655e-06,
+      "loss": 1.7463,
+      "step": 1671
+    },
+    {
+      "epoch": 1.8323287671232875,
+      "grad_norm": 2.3553171157836914,
+      "learning_rate": 2.1913078797725427e-06,
+      "loss": 1.7844,
+      "step": 1672
+    },
+    {
+      "epoch": 1.8334246575342465,
+      "grad_norm": 2.3724591732025146,
+      "learning_rate": 2.18927701056052e-06,
+      "loss": 1.7955,
+      "step": 1673
+    },
+    {
+      "epoch": 1.8345205479452056,
+      "grad_norm": 2.4300284385681152,
+      "learning_rate": 2.1872461413484973e-06,
+      "loss": 1.8697,
+      "step": 1674
+    },
+    {
+      "epoch": 1.8356164383561644,
+      "grad_norm": 2.4300284385681152,
+      "learning_rate": 2.1872461413484973e-06,
+      "loss": 1.8065,
+      "step": 1675
+    },
+    {
+      "epoch": 1.8367123287671232,
+      "grad_norm": 2.523627519607544,
+      "learning_rate": 2.1852152721364746e-06,
+      "loss": 1.801,
+      "step": 1676
+    },
+    {
+      "epoch": 1.8378082191780822,
+      "grad_norm": 2.2884891033172607,
+      "learning_rate": 2.183184402924452e-06,
+      "loss": 1.7512,
+      "step": 1677
+    },
+    {
+      "epoch": 1.838904109589041,
+      "grad_norm": 2.3114986419677734,
+      "learning_rate": 2.181153533712429e-06,
+      "loss": 1.8245,
+      "step": 1678
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 2.326855421066284,
+      "learning_rate": 2.1791226645004064e-06,
+      "loss": 1.7921,
+      "step": 1679
+    },
+    {
+      "epoch": 1.841095890410959,
+      "grad_norm": 2.3448753356933594,
+      "learning_rate": 2.1770917952883837e-06,
+      "loss": 1.7709,
+      "step": 1680
+    },
+    {
+      "epoch": 1.841095890410959,
+      "eval_loss": 1.996195673942566,
+      "eval_runtime": 37.0481,
+      "eval_samples_per_second": 26.992,
+      "eval_steps_per_second": 1.134,
+      "step": 1680
+    },
+    {
+      "epoch": 1.842191780821918,
+      "grad_norm": 2.223470687866211,
+      "learning_rate": 2.175060926076361e-06,
+      "loss": 1.8468,
+      "step": 1681
+    },
+    {
+      "epoch": 1.8432876712328767,
+      "grad_norm": 2.29726505279541,
+      "learning_rate": 2.1730300568643382e-06,
+      "loss": 1.8753,
+      "step": 1682
+    },
+    {
+      "epoch": 1.8443835616438355,
+      "grad_norm": 2.3869571685791016,
+      "learning_rate": 2.1709991876523155e-06,
+      "loss": 1.7835,
+      "step": 1683
+    },
+    {
+      "epoch": 1.8454794520547946,
+      "grad_norm": 2.192608594894409,
+      "learning_rate": 2.1689683184402928e-06,
+      "loss": 1.7556,
+      "step": 1684
+    },
+    {
+      "epoch": 1.8465753424657534,
+      "grad_norm": 2.355975866317749,
+      "learning_rate": 2.16693744922827e-06,
+      "loss": 1.8323,
+      "step": 1685
+    },
+    {
+      "epoch": 1.8476712328767122,
+      "grad_norm": 2.243938446044922,
+      "learning_rate": 2.1649065800162473e-06,
+      "loss": 1.7942,
+      "step": 1686
+    },
+    {
+      "epoch": 1.8487671232876712,
+      "grad_norm": 2.337069511413574,
+      "learning_rate": 2.1628757108042246e-06,
+      "loss": 1.7709,
+      "step": 1687
+    },
+    {
+      "epoch": 1.8498630136986303,
+      "grad_norm": 2.257504463195801,
+      "learning_rate": 2.160844841592202e-06,
+      "loss": 1.7496,
+      "step": 1688
+    },
+    {
+      "epoch": 1.850958904109589,
+      "grad_norm": 2.287336826324463,
+      "learning_rate": 2.1588139723801787e-06,
+      "loss": 1.7454,
+      "step": 1689
+    },
+    {
+      "epoch": 1.8520547945205479,
+      "grad_norm": 2.2864668369293213,
+      "learning_rate": 2.156783103168156e-06,
+      "loss": 1.7807,
+      "step": 1690
+    },
+    {
+      "epoch": 1.8520547945205479,
+      "eval_loss": 1.9940879344940186,
+      "eval_runtime": 37.0261,
+      "eval_samples_per_second": 27.008,
+      "eval_steps_per_second": 1.134,
+      "step": 1690
+    },
+    {
+      "epoch": 1.853150684931507,
+      "grad_norm": 2.1782145500183105,
+      "learning_rate": 2.1547522339561333e-06,
+      "loss": 1.7567,
+      "step": 1691
+    },
+    {
+      "epoch": 1.8542465753424657,
+      "grad_norm": 2.252023458480835,
+      "learning_rate": 2.1527213647441106e-06,
+      "loss": 1.811,
+      "step": 1692
+    },
+    {
+      "epoch": 1.8553424657534245,
+      "grad_norm": 2.2532849311828613,
+      "learning_rate": 2.150690495532088e-06,
+      "loss": 1.8491,
+      "step": 1693
+    },
+    {
+      "epoch": 1.8564383561643836,
+      "grad_norm": 2.35648250579834,
+      "learning_rate": 2.148659626320065e-06,
+      "loss": 1.8125,
+      "step": 1694
+    },
+    {
+      "epoch": 1.8575342465753426,
+      "grad_norm": 2.3490521907806396,
+      "learning_rate": 2.1466287571080424e-06,
+      "loss": 1.8035,
+      "step": 1695
+    },
+    {
+      "epoch": 1.8586301369863014,
+      "grad_norm": 2.2851879596710205,
+      "learning_rate": 2.1445978878960197e-06,
+      "loss": 1.7755,
+      "step": 1696
+    },
+    {
+      "epoch": 1.8597260273972602,
+      "grad_norm": 2.166754961013794,
+      "learning_rate": 2.142567018683997e-06,
+      "loss": 1.7861,
+      "step": 1697
+    },
+    {
+      "epoch": 1.8608219178082193,
+      "grad_norm": 2.2356410026550293,
+      "learning_rate": 2.140536149471974e-06,
+      "loss": 1.8412,
+      "step": 1698
+    },
+    {
+      "epoch": 1.861917808219178,
+      "grad_norm": 2.215618133544922,
+      "learning_rate": 2.1385052802599515e-06,
+      "loss": 1.7623,
+      "step": 1699
+    },
+    {
+      "epoch": 1.8630136986301369,
+      "grad_norm": 2.283298969268799,
+      "learning_rate": 2.1364744110479288e-06,
+      "loss": 1.7692,
+      "step": 1700
+    },
+    {
+      "epoch": 1.8630136986301369,
+      "eval_loss": 1.9961501359939575,
+      "eval_runtime": 37.0338,
+      "eval_samples_per_second": 27.002,
+      "eval_steps_per_second": 1.134,
+      "step": 1700
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2736,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.299425995915264e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b7adbe5a2dbf27f05befec6cc65fbc93e90eefd0
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa882279512a8c278376c0c601206933f3c5662ddcd3c13c53bf32ae3680d14
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-1700/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb0f9a53a507ffcdb60410deb71d60ef801bf350
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/config.json
@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/generation_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c389b841a9a29ddd904e02d1eb0e08dcce82ad9
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.45.1"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/latest b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/latest
new file mode 100644
index 0000000000000000000000000000000000000000..4b10acccf3e8395339ff8799cea202bbc54d7f7d
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/latest
@@ -0,0 +1 @@
+global_step900
\ No newline at end of file
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/model.safetensors.index.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed64de846d720b9a7859dc20575fea8e8ca51940
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/model.safetensors.index.json
@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_size": 7213504512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_0.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5a018a68bd5303fcda57dcb6093948dc236b697a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34bfd29981738a04d7cb9b0b31f032749b5bd5fa1b120104d856a6f5b060af88
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_1.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..85d20c3dd29d7f71371d2ce18d57829a2ebd22bc
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9e7ca9eed1d152be37852a8850dde9294edfaf37097b42c75c54b278336ef61
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_2.pth b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_2.pth
new file mode 100644
index 0000000000000000000000000000000000000000..137d87c143da2b66811890c8cf1e8b013ef6eacf
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/rng_state_2.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c45c6121ed4cfe9395ee5153d26d376e4f02f8dfa1bbceafb8a5416efcdd14
+size 16567
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/scheduler.pt b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86063610918de03fe10ff5915f245b21a80f8845
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:977be84d836c6e1ba1cd1d60ad9c4b8ba9f4c0a63264746f477be2406b5629d7
+size 627
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/special_tokens_map.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..6949c5975ee0e961ef61cf31010dce04df0a03f8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..309789d5ccd06f418fbcfb7472a6a9affda68101
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d422bc7d90212ffb8070a743cdbe8ff2dfc5634b1fce8f2e3fa671ee441989eb
+size 17210374
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer_config.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5684b14e2756b56aa397c1324e239b442162b34a
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/tokenizer_config.json
@@ -0,0 +1,2070 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "[PAD]",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/trainer_state.json b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..cf7c179079f89c4e5d5273d30de157546c41f77b
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/trainer_state.json
@@ -0,0 +1,7053 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9863013698630136,
+  "eval_steps": 10,
+  "global_step": 900,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001095890410958904,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0636,
+      "step": 1
+    },
+    {
+      "epoch": 0.002191780821917808,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0501,
+      "step": 2
+    },
+    {
+      "epoch": 0.0032876712328767125,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0238,
+      "step": 3
+    },
+    {
+      "epoch": 0.004383561643835616,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0346,
+      "step": 4
+    },
+    {
+      "epoch": 0.005479452054794521,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0108,
+      "step": 5
+    },
+    {
+      "epoch": 0.006575342465753425,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.131,
+      "step": 6
+    },
+    {
+      "epoch": 0.007671232876712329,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0488,
+      "step": 7
+    },
+    {
+      "epoch": 0.008767123287671232,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0905,
+      "step": 8
+    },
+    {
+      "epoch": 0.009863013698630137,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0493,
+      "step": 9
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.1038,
+      "step": 10
+    },
+    {
+      "epoch": 0.010958904109589041,
+      "eval_loss": 3.100078821182251,
+      "eval_runtime": 37.5838,
+      "eval_samples_per_second": 26.607,
+      "eval_steps_per_second": 1.118,
+      "step": 10
+    },
+    {
+      "epoch": 0.012054794520547946,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0804,
+      "step": 11
+    },
+    {
+      "epoch": 0.01315068493150685,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0053,
+      "step": 12
+    },
+    {
+      "epoch": 0.014246575342465753,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0607,
+      "step": 13
+    },
+    {
+      "epoch": 0.015342465753424657,
+      "grad_norm": 0.0,
+      "learning_rate": 0.0,
+      "loss": 3.0539,
+      "step": 14
+    },
+    {
+      "epoch": 0.01643835616438356,
+      "grad_norm": 5.420984745025635,
+      "learning_rate": 1.8248175182481753e-08,
+      "loss": 3.0941,
+      "step": 15
+    },
+    {
+      "epoch": 0.017534246575342465,
+      "grad_norm": 5.76840877532959,
+      "learning_rate": 3.6496350364963505e-08,
+      "loss": 3.036,
+      "step": 16
+    },
+    {
+      "epoch": 0.01863013698630137,
+      "grad_norm": 5.905357360839844,
+      "learning_rate": 5.4744525547445264e-08,
+      "loss": 3.0491,
+      "step": 17
+    },
+    {
+      "epoch": 0.019726027397260273,
+      "grad_norm": 5.981553554534912,
+      "learning_rate": 7.299270072992701e-08,
+      "loss": 3.0775,
+      "step": 18
+    },
+    {
+      "epoch": 0.020821917808219178,
+      "grad_norm": 6.240172863006592,
+      "learning_rate": 9.124087591240876e-08,
+      "loss": 3.0345,
+      "step": 19
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "grad_norm": 5.59375,
+      "learning_rate": 1.0948905109489053e-07,
+      "loss": 3.1063,
+      "step": 20
+    },
+    {
+      "epoch": 0.021917808219178082,
+      "eval_loss": 3.098820686340332,
+      "eval_runtime": 37.2695,
+      "eval_samples_per_second": 26.832,
+      "eval_steps_per_second": 1.127,
+      "step": 20
+    },
+    {
+      "epoch": 0.023013698630136987,
+      "grad_norm": 5.819795608520508,
+      "learning_rate": 1.2773722627737227e-07,
+      "loss": 3.0808,
+      "step": 21
+    },
+    {
+      "epoch": 0.02410958904109589,
+      "grad_norm": 5.7714762687683105,
+      "learning_rate": 1.4598540145985402e-07,
+      "loss": 3.0055,
+      "step": 22
+    },
+    {
+      "epoch": 0.025205479452054796,
+      "grad_norm": 5.51552152633667,
+      "learning_rate": 1.642335766423358e-07,
+      "loss": 3.0396,
+      "step": 23
+    },
+    {
+      "epoch": 0.0263013698630137,
+      "grad_norm": 5.594012260437012,
+      "learning_rate": 1.8248175182481753e-07,
+      "loss": 3.0824,
+      "step": 24
+    },
+    {
+      "epoch": 0.0273972602739726,
+      "grad_norm": 5.1974029541015625,
+      "learning_rate": 2.0072992700729928e-07,
+      "loss": 3.08,
+      "step": 25
+    },
+    {
+      "epoch": 0.028493150684931506,
+      "grad_norm": 4.733968257904053,
+      "learning_rate": 2.1897810218978106e-07,
+      "loss": 3.0597,
+      "step": 26
+    },
+    {
+      "epoch": 0.02958904109589041,
+      "grad_norm": 4.717319011688232,
+      "learning_rate": 2.3722627737226278e-07,
+      "loss": 3.0696,
+      "step": 27
+    },
+    {
+      "epoch": 0.030684931506849315,
+      "grad_norm": 4.6303534507751465,
+      "learning_rate": 2.5547445255474454e-07,
+      "loss": 3.0273,
+      "step": 28
+    },
+    {
+      "epoch": 0.03178082191780822,
+      "grad_norm": 4.413999557495117,
+      "learning_rate": 2.737226277372263e-07,
+      "loss": 3.0202,
+      "step": 29
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "grad_norm": 4.516043186187744,
+      "learning_rate": 2.9197080291970804e-07,
+      "loss": 2.9926,
+      "step": 30
+    },
+    {
+      "epoch": 0.03287671232876712,
+      "eval_loss": 3.0561797618865967,
+      "eval_runtime": 37.0351,
+      "eval_samples_per_second": 27.001,
+      "eval_steps_per_second": 1.134,
+      "step": 30
+    },
+    {
+      "epoch": 0.03397260273972603,
+      "grad_norm": 3.7179207801818848,
+      "learning_rate": 3.102189781021898e-07,
+      "loss": 3.0086,
+      "step": 31
+    },
+    {
+      "epoch": 0.03506849315068493,
+      "grad_norm": 3.7840425968170166,
+      "learning_rate": 3.284671532846716e-07,
+      "loss": 2.9986,
+      "step": 32
+    },
+    {
+      "epoch": 0.03616438356164384,
+      "grad_norm": 3.843116521835327,
+      "learning_rate": 3.467153284671533e-07,
+      "loss": 2.9677,
+      "step": 33
+    },
+    {
+      "epoch": 0.03726027397260274,
+      "grad_norm": 3.8528714179992676,
+      "learning_rate": 3.6496350364963505e-07,
+      "loss": 2.9732,
+      "step": 34
+    },
+    {
+      "epoch": 0.038356164383561646,
+      "grad_norm": 3.76963210105896,
+      "learning_rate": 3.8321167883211686e-07,
+      "loss": 2.9855,
+      "step": 35
+    },
+    {
+      "epoch": 0.03945205479452055,
+      "grad_norm": 3.6240015029907227,
+      "learning_rate": 4.0145985401459856e-07,
+      "loss": 3.0117,
+      "step": 36
+    },
+    {
+      "epoch": 0.040547945205479455,
+      "grad_norm": 3.1380624771118164,
+      "learning_rate": 4.197080291970803e-07,
+      "loss": 2.9481,
+      "step": 37
+    },
+    {
+      "epoch": 0.041643835616438356,
+      "grad_norm": 3.565329074859619,
+      "learning_rate": 4.379562043795621e-07,
+      "loss": 2.9175,
+      "step": 38
+    },
+    {
+      "epoch": 0.042739726027397264,
+      "grad_norm": 3.8062448501586914,
+      "learning_rate": 4.562043795620438e-07,
+      "loss": 3.0074,
+      "step": 39
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "grad_norm": 3.412046432495117,
+      "learning_rate": 4.7445255474452557e-07,
+      "loss": 2.9119,
+      "step": 40
+    },
+    {
+      "epoch": 0.043835616438356165,
+      "eval_loss": 2.9874370098114014,
+      "eval_runtime": 37.0514,
+      "eval_samples_per_second": 26.99,
+      "eval_steps_per_second": 1.134,
+      "step": 40
+    },
+    {
+      "epoch": 0.044931506849315066,
+      "grad_norm": 3.227437973022461,
+      "learning_rate": 4.927007299270073e-07,
+      "loss": 2.9436,
+      "step": 41
+    },
+    {
+      "epoch": 0.046027397260273974,
+      "grad_norm": 2.9633443355560303,
+      "learning_rate": 5.109489051094891e-07,
+      "loss": 2.967,
+      "step": 42
+    },
+    {
+      "epoch": 0.047123287671232875,
+      "grad_norm": 2.942678928375244,
+      "learning_rate": 5.291970802919708e-07,
+      "loss": 2.9375,
+      "step": 43
+    },
+    {
+      "epoch": 0.04821917808219178,
+      "grad_norm": 2.9806888103485107,
+      "learning_rate": 5.474452554744526e-07,
+      "loss": 2.9474,
+      "step": 44
+    },
+    {
+      "epoch": 0.049315068493150684,
+      "grad_norm": 3.000063419342041,
+      "learning_rate": 5.656934306569343e-07,
+      "loss": 2.9289,
+      "step": 45
+    },
+    {
+      "epoch": 0.05041095890410959,
+      "grad_norm": 3.1481194496154785,
+      "learning_rate": 5.839416058394161e-07,
+      "loss": 2.8869,
+      "step": 46
+    },
+    {
+      "epoch": 0.05150684931506849,
+      "grad_norm": 2.949202060699463,
+      "learning_rate": 6.021897810218979e-07,
+      "loss": 2.8783,
+      "step": 47
+    },
+    {
+      "epoch": 0.0526027397260274,
+      "grad_norm": 2.706526279449463,
+      "learning_rate": 6.204379562043796e-07,
+      "loss": 2.9024,
+      "step": 48
+    },
+    {
+      "epoch": 0.0536986301369863,
+      "grad_norm": 2.7317655086517334,
+      "learning_rate": 6.386861313868614e-07,
+      "loss": 2.8325,
+      "step": 49
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "grad_norm": 2.25644588470459,
+      "learning_rate": 6.569343065693432e-07,
+      "loss": 2.9409,
+      "step": 50
+    },
+    {
+      "epoch": 0.0547945205479452,
+      "eval_loss": 2.909320116043091,
+      "eval_runtime": 37.0714,
+      "eval_samples_per_second": 26.975,
+      "eval_steps_per_second": 1.133,
+      "step": 50
+    },
+    {
+      "epoch": 0.05589041095890411,
+      "grad_norm": 2.8789613246917725,
+      "learning_rate": 6.751824817518248e-07,
+      "loss": 2.8459,
+      "step": 51
+    },
+    {
+      "epoch": 0.05698630136986301,
+      "grad_norm": 2.686873435974121,
+      "learning_rate": 6.934306569343066e-07,
+      "loss": 2.8653,
+      "step": 52
+    },
+    {
+      "epoch": 0.05808219178082192,
+      "grad_norm": 2.54382586479187,
+      "learning_rate": 7.116788321167883e-07,
+      "loss": 2.7973,
+      "step": 53
+    },
+    {
+      "epoch": 0.05917808219178082,
+      "grad_norm": 2.1871724128723145,
+      "learning_rate": 7.299270072992701e-07,
+      "loss": 2.7982,
+      "step": 54
+    },
+    {
+      "epoch": 0.06027397260273973,
+      "grad_norm": 2.145303249359131,
+      "learning_rate": 7.48175182481752e-07,
+      "loss": 2.8195,
+      "step": 55
+    },
+    {
+      "epoch": 0.06136986301369863,
+      "grad_norm": 2.1635069847106934,
+      "learning_rate": 7.664233576642337e-07,
+      "loss": 2.825,
+      "step": 56
+    },
+    {
+      "epoch": 0.06246575342465754,
+      "grad_norm": 2.3530752658843994,
+      "learning_rate": 7.846715328467154e-07,
+      "loss": 2.8407,
+      "step": 57
+    },
+    {
+      "epoch": 0.06356164383561644,
+      "grad_norm": 2.2455942630767822,
+      "learning_rate": 8.029197080291971e-07,
+      "loss": 2.8182,
+      "step": 58
+    },
+    {
+      "epoch": 0.06465753424657535,
+      "grad_norm": 1.8985673189163208,
+      "learning_rate": 8.211678832116789e-07,
+      "loss": 2.772,
+      "step": 59
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "grad_norm": 1.9699499607086182,
+      "learning_rate": 8.394160583941606e-07,
+      "loss": 2.7367,
+      "step": 60
+    },
+    {
+      "epoch": 0.06575342465753424,
+      "eval_loss": 2.8451859951019287,
+      "eval_runtime": 37.0793,
+      "eval_samples_per_second": 26.969,
+      "eval_steps_per_second": 1.133,
+      "step": 60
+    },
+    {
+      "epoch": 0.06684931506849315,
+      "grad_norm": 2.159547805786133,
+      "learning_rate": 8.576642335766425e-07,
+      "loss": 2.7635,
+      "step": 61
+    },
+    {
+      "epoch": 0.06794520547945206,
+      "grad_norm": 2.12705659866333,
+      "learning_rate": 8.759124087591242e-07,
+      "loss": 2.7624,
+      "step": 62
+    },
+    {
+      "epoch": 0.06904109589041096,
+      "grad_norm": 1.8393241167068481,
+      "learning_rate": 8.94160583941606e-07,
+      "loss": 2.7763,
+      "step": 63
+    },
+    {
+      "epoch": 0.07013698630136986,
+      "grad_norm": 1.683522343635559,
+      "learning_rate": 9.124087591240876e-07,
+      "loss": 2.7762,
+      "step": 64
+    },
+    {
+      "epoch": 0.07123287671232877,
+      "grad_norm": 1.9260376691818237,
+      "learning_rate": 9.306569343065694e-07,
+      "loss": 2.7418,
+      "step": 65
+    },
+    {
+      "epoch": 0.07232876712328767,
+      "grad_norm": 2.09322190284729,
+      "learning_rate": 9.489051094890511e-07,
+      "loss": 2.7625,
+      "step": 66
+    },
+    {
+      "epoch": 0.07342465753424658,
+      "grad_norm": 1.9183429479599,
+      "learning_rate": 9.67153284671533e-07,
+      "loss": 2.7453,
+      "step": 67
+    },
+    {
+      "epoch": 0.07452054794520548,
+      "grad_norm": 1.9792778491973877,
+      "learning_rate": 9.854014598540146e-07,
+      "loss": 2.7858,
+      "step": 68
+    },
+    {
+      "epoch": 0.07561643835616438,
+      "grad_norm": 2.1397311687469482,
+      "learning_rate": 1.0036496350364965e-06,
+      "loss": 2.7242,
+      "step": 69
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "grad_norm": 1.8556463718414307,
+      "learning_rate": 1.0218978102189781e-06,
+      "loss": 2.6853,
+      "step": 70
+    },
+    {
+      "epoch": 0.07671232876712329,
+      "eval_loss": 2.7787046432495117,
+      "eval_runtime": 37.1017,
+      "eval_samples_per_second": 26.953,
+      "eval_steps_per_second": 1.132,
+      "step": 70
+    },
+    {
+      "epoch": 0.07780821917808219,
+      "grad_norm": 1.9776800870895386,
+      "learning_rate": 1.04014598540146e-06,
+      "loss": 2.749,
+      "step": 71
+    },
+    {
+      "epoch": 0.0789041095890411,
+      "grad_norm": 2.3972630500793457,
+      "learning_rate": 1.0583941605839416e-06,
+      "loss": 2.6457,
+      "step": 72
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 2.090210199356079,
+      "learning_rate": 1.0766423357664233e-06,
+      "loss": 2.6401,
+      "step": 73
+    },
+    {
+      "epoch": 0.08109589041095891,
+      "grad_norm": 2.4117701053619385,
+      "learning_rate": 1.0948905109489052e-06,
+      "loss": 2.6649,
+      "step": 74
+    },
+    {
+      "epoch": 0.0821917808219178,
+      "grad_norm": 2.4845173358917236,
+      "learning_rate": 1.113138686131387e-06,
+      "loss": 2.6855,
+      "step": 75
+    },
+    {
+      "epoch": 0.08328767123287671,
+      "grad_norm": 1.7779242992401123,
+      "learning_rate": 1.1313868613138687e-06,
+      "loss": 2.6926,
+      "step": 76
+    },
+    {
+      "epoch": 0.08438356164383562,
+      "grad_norm": 2.3163535594940186,
+      "learning_rate": 1.1496350364963505e-06,
+      "loss": 2.6302,
+      "step": 77
+    },
+    {
+      "epoch": 0.08547945205479453,
+      "grad_norm": 2.2296528816223145,
+      "learning_rate": 1.1678832116788322e-06,
+      "loss": 2.6054,
+      "step": 78
+    },
+    {
+      "epoch": 0.08657534246575342,
+      "grad_norm": 2.253431797027588,
+      "learning_rate": 1.1861313868613138e-06,
+      "loss": 2.6104,
+      "step": 79
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "grad_norm": 2.490609645843506,
+      "learning_rate": 1.2043795620437959e-06,
+      "loss": 2.6078,
+      "step": 80
+    },
+    {
+      "epoch": 0.08767123287671233,
+      "eval_loss": 2.706103563308716,
+      "eval_runtime": 37.0892,
+      "eval_samples_per_second": 26.962,
+      "eval_steps_per_second": 1.132,
+      "step": 80
+    },
+    {
+      "epoch": 0.08876712328767124,
+      "grad_norm": 2.3048248291015625,
+      "learning_rate": 1.2226277372262775e-06,
+      "loss": 2.6096,
+      "step": 81
+    },
+    {
+      "epoch": 0.08986301369863013,
+      "grad_norm": 2.440666913986206,
+      "learning_rate": 1.2408759124087592e-06,
+      "loss": 2.6412,
+      "step": 82
+    },
+    {
+      "epoch": 0.09095890410958904,
+      "grad_norm": 1.9456374645233154,
+      "learning_rate": 1.259124087591241e-06,
+      "loss": 2.591,
+      "step": 83
+    },
+    {
+      "epoch": 0.09205479452054795,
+      "grad_norm": 2.598172664642334,
+      "learning_rate": 1.2773722627737229e-06,
+      "loss": 2.6093,
+      "step": 84
+    },
+    {
+      "epoch": 0.09315068493150686,
+      "grad_norm": 2.2587032318115234,
+      "learning_rate": 1.2956204379562045e-06,
+      "loss": 2.5949,
+      "step": 85
+    },
+    {
+      "epoch": 0.09424657534246575,
+      "grad_norm": 2.2927448749542236,
+      "learning_rate": 1.3138686131386864e-06,
+      "loss": 2.5464,
+      "step": 86
+    },
+    {
+      "epoch": 0.09534246575342466,
+      "grad_norm": 2.5940468311309814,
+      "learning_rate": 1.332116788321168e-06,
+      "loss": 2.5613,
+      "step": 87
+    },
+    {
+      "epoch": 0.09643835616438357,
+      "grad_norm": 2.073207378387451,
+      "learning_rate": 1.3503649635036497e-06,
+      "loss": 2.4955,
+      "step": 88
+    },
+    {
+      "epoch": 0.09753424657534246,
+      "grad_norm": 3.211766242980957,
+      "learning_rate": 1.3686131386861315e-06,
+      "loss": 2.5315,
+      "step": 89
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "grad_norm": 2.2590038776397705,
+      "learning_rate": 1.3868613138686132e-06,
+      "loss": 2.4641,
+      "step": 90
+    },
+    {
+      "epoch": 0.09863013698630137,
+      "eval_loss": 2.625087261199951,
+      "eval_runtime": 37.0852,
+      "eval_samples_per_second": 26.965,
+      "eval_steps_per_second": 1.133,
+      "step": 90
+    },
+    {
+      "epoch": 0.09972602739726028,
+      "grad_norm": 2.3959836959838867,
+      "learning_rate": 1.405109489051095e-06,
+      "loss": 2.5537,
+      "step": 91
+    },
+    {
+      "epoch": 0.10082191780821918,
+      "grad_norm": 2.596761465072632,
+      "learning_rate": 1.4233576642335767e-06,
+      "loss": 2.5135,
+      "step": 92
+    },
+    {
+      "epoch": 0.10191780821917808,
+      "grad_norm": 2.475759267807007,
+      "learning_rate": 1.4416058394160583e-06,
+      "loss": 2.5436,
+      "step": 93
+    },
+    {
+      "epoch": 0.10301369863013699,
+      "grad_norm": 3.0613298416137695,
+      "learning_rate": 1.4598540145985402e-06,
+      "loss": 2.5066,
+      "step": 94
+    },
+    {
+      "epoch": 0.10410958904109589,
+      "grad_norm": 2.406097650527954,
+      "learning_rate": 1.478102189781022e-06,
+      "loss": 2.4646,
+      "step": 95
+    },
+    {
+      "epoch": 0.1052054794520548,
+      "grad_norm": 2.5872504711151123,
+      "learning_rate": 1.496350364963504e-06,
+      "loss": 2.5351,
+      "step": 96
+    },
+    {
+      "epoch": 0.1063013698630137,
+      "grad_norm": 2.4294471740722656,
+      "learning_rate": 1.5145985401459856e-06,
+      "loss": 2.4953,
+      "step": 97
+    },
+    {
+      "epoch": 0.1073972602739726,
+      "grad_norm": 2.674884080886841,
+      "learning_rate": 1.5328467153284674e-06,
+      "loss": 2.5133,
+      "step": 98
+    },
+    {
+      "epoch": 0.10849315068493151,
+      "grad_norm": 2.5096852779388428,
+      "learning_rate": 1.551094890510949e-06,
+      "loss": 2.4693,
+      "step": 99
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "grad_norm": 2.7390334606170654,
+      "learning_rate": 1.5693430656934307e-06,
+      "loss": 2.4195,
+      "step": 100
+    },
+    {
+      "epoch": 0.1095890410958904,
+      "eval_loss": 2.55997371673584,
+      "eval_runtime": 37.0732,
+      "eval_samples_per_second": 26.974,
+      "eval_steps_per_second": 1.133,
+      "step": 100
+    },
+    {
+      "epoch": 0.11068493150684931,
+      "grad_norm": 2.8887312412261963,
+      "learning_rate": 1.5875912408759126e-06,
+      "loss": 2.4193,
+      "step": 101
+    },
+    {
+      "epoch": 0.11178082191780822,
+      "grad_norm": 3.0721421241760254,
+      "learning_rate": 1.6058394160583942e-06,
+      "loss": 2.4709,
+      "step": 102
+    },
+    {
+      "epoch": 0.11287671232876713,
+      "grad_norm": 2.728156566619873,
+      "learning_rate": 1.624087591240876e-06,
+      "loss": 2.5105,
+      "step": 103
+    },
+    {
+      "epoch": 0.11397260273972602,
+      "grad_norm": 3.1214346885681152,
+      "learning_rate": 1.6423357664233577e-06,
+      "loss": 2.4316,
+      "step": 104
+    },
+    {
+      "epoch": 0.11506849315068493,
+      "grad_norm": 2.9249517917633057,
+      "learning_rate": 1.6605839416058394e-06,
+      "loss": 2.3737,
+      "step": 105
+    },
+    {
+      "epoch": 0.11616438356164384,
+      "grad_norm": 2.6646173000335693,
+      "learning_rate": 1.6788321167883212e-06,
+      "loss": 2.4165,
+      "step": 106
+    },
+    {
+      "epoch": 0.11726027397260275,
+      "grad_norm": 3.108067750930786,
+      "learning_rate": 1.6970802919708029e-06,
+      "loss": 2.4217,
+      "step": 107
+    },
+    {
+      "epoch": 0.11835616438356164,
+      "grad_norm": 3.142338991165161,
+      "learning_rate": 1.715328467153285e-06,
+      "loss": 2.3527,
+      "step": 108
+    },
+    {
+      "epoch": 0.11945205479452055,
+      "grad_norm": 2.9078011512756348,
+      "learning_rate": 1.7335766423357666e-06,
+      "loss": 2.3927,
+      "step": 109
+    },
+    {
+      "epoch": 0.12054794520547946,
+      "grad_norm": 3.40766978263855,
+      "learning_rate": 1.7518248175182485e-06,
+      "loss": 2.4021,
+      "step": 110
+    },
+    {
+      "epoch": 0.12054794520547946,
+      "eval_loss": 2.5031485557556152,
+      "eval_runtime": 36.8147,
+      "eval_samples_per_second": 27.163,
+      "eval_steps_per_second": 1.141,
+      "step": 110
+    },
+    {
+      "epoch": 0.12164383561643835,
+      "grad_norm": 4.396062850952148,
+      "learning_rate": 1.77007299270073e-06,
+      "loss": 2.4373,
+      "step": 111
+    },
+    {
+      "epoch": 0.12273972602739726,
+      "grad_norm": 4.0441365242004395,
+      "learning_rate": 1.788321167883212e-06,
+      "loss": 2.4162,
+      "step": 112
+    },
+    {
+      "epoch": 0.12383561643835617,
+      "grad_norm": 2.8869969844818115,
+      "learning_rate": 1.8065693430656936e-06,
+      "loss": 2.4515,
+      "step": 113
+    },
+    {
+      "epoch": 0.12493150684931507,
+      "grad_norm": 3.4833669662475586,
+      "learning_rate": 1.8248175182481753e-06,
+      "loss": 2.3965,
+      "step": 114
+    },
+    {
+      "epoch": 0.12602739726027398,
+      "grad_norm": 2.9996068477630615,
+      "learning_rate": 1.8430656934306571e-06,
+      "loss": 2.3374,
+      "step": 115
+    },
+    {
+      "epoch": 0.12712328767123288,
+      "grad_norm": 2.701371192932129,
+      "learning_rate": 1.8613138686131388e-06,
+      "loss": 2.3608,
+      "step": 116
+    },
+    {
+      "epoch": 0.12821917808219177,
+      "grad_norm": 2.9491288661956787,
+      "learning_rate": 1.8795620437956206e-06,
+      "loss": 2.355,
+      "step": 117
+    },
+    {
+      "epoch": 0.1293150684931507,
+      "grad_norm": 2.9266560077667236,
+      "learning_rate": 1.8978102189781023e-06,
+      "loss": 2.3475,
+      "step": 118
+    },
+    {
+      "epoch": 0.1304109589041096,
+      "grad_norm": 3.0419797897338867,
+      "learning_rate": 1.916058394160584e-06,
+      "loss": 2.3475,
+      "step": 119
+    },
+    {
+      "epoch": 0.13150684931506848,
+      "grad_norm": 2.4531214237213135,
+      "learning_rate": 1.934306569343066e-06,
+      "loss": 2.3724,
+      "step": 120
+    },
+    {
+      "epoch": 0.13150684931506848,
+      "eval_loss": 2.455423593521118,
+      "eval_runtime": 37.0304,
+      "eval_samples_per_second": 27.005,
+      "eval_steps_per_second": 1.134,
+      "step": 120
+    },
+    {
+      "epoch": 0.1326027397260274,
+      "grad_norm": 3.572502851486206,
+      "learning_rate": 1.952554744525548e-06,
+      "loss": 2.2671,
+      "step": 121
+    },
+    {
+      "epoch": 0.1336986301369863,
+      "grad_norm": 3.5408504009246826,
+      "learning_rate": 1.9708029197080293e-06,
+      "loss": 2.3133,
+      "step": 122
+    },
+    {
+      "epoch": 0.13479452054794522,
+      "grad_norm": 2.488725423812866,
+      "learning_rate": 1.989051094890511e-06,
+      "loss": 2.3141,
+      "step": 123
+    },
+    {
+      "epoch": 0.1358904109589041,
+      "grad_norm": 3.241276264190674,
+      "learning_rate": 2.007299270072993e-06,
+      "loss": 2.318,
+      "step": 124
+    },
+    {
+      "epoch": 0.136986301369863,
+      "grad_norm": 2.853959798812866,
+      "learning_rate": 2.025547445255475e-06,
+      "loss": 2.3712,
+      "step": 125
+    },
+    {
+      "epoch": 0.13808219178082193,
+      "grad_norm": 2.9798169136047363,
+      "learning_rate": 2.0437956204379563e-06,
+      "loss": 2.3072,
+      "step": 126
+    },
+    {
+      "epoch": 0.13917808219178082,
+      "grad_norm": 2.8529601097106934,
+      "learning_rate": 2.062043795620438e-06,
+      "loss": 2.318,
+      "step": 127
+    },
+    {
+      "epoch": 0.14027397260273972,
+      "grad_norm": 3.475337028503418,
+      "learning_rate": 2.08029197080292e-06,
+      "loss": 2.31,
+      "step": 128
+    },
+    {
+      "epoch": 0.14136986301369864,
+      "grad_norm": 4.213955879211426,
+      "learning_rate": 2.0985401459854014e-06,
+      "loss": 2.3734,
+      "step": 129
+    },
+    {
+      "epoch": 0.14246575342465753,
+      "grad_norm": 3.0833330154418945,
+      "learning_rate": 2.1167883211678833e-06,
+      "loss": 2.2602,
+      "step": 130
+    },
+    {
+      "epoch": 0.14246575342465753,
+      "eval_loss": 2.4205682277679443,
+      "eval_runtime": 37.0798,
+      "eval_samples_per_second": 26.969,
+      "eval_steps_per_second": 1.133,
+      "step": 130
+    },
+    {
+      "epoch": 0.14356164383561643,
+      "grad_norm": 3.411235809326172,
+      "learning_rate": 2.135036496350365e-06,
+      "loss": 2.3255,
+      "step": 131
+    },
+    {
+      "epoch": 0.14465753424657535,
+      "grad_norm": 2.882053852081299,
+      "learning_rate": 2.1532846715328466e-06,
+      "loss": 2.3483,
+      "step": 132
+    },
+    {
+      "epoch": 0.14575342465753424,
+      "grad_norm": 3.2436420917510986,
+      "learning_rate": 2.171532846715329e-06,
+      "loss": 2.2492,
+      "step": 133
+    },
+    {
+      "epoch": 0.14684931506849316,
+      "grad_norm": 2.929466962814331,
+      "learning_rate": 2.1897810218978103e-06,
+      "loss": 2.2647,
+      "step": 134
+    },
+    {
+      "epoch": 0.14794520547945206,
+      "grad_norm": 2.9344966411590576,
+      "learning_rate": 2.208029197080292e-06,
+      "loss": 2.3061,
+      "step": 135
+    },
+    {
+      "epoch": 0.14904109589041095,
+      "grad_norm": 3.2567203044891357,
+      "learning_rate": 2.226277372262774e-06,
+      "loss": 2.2746,
+      "step": 136
+    },
+    {
+      "epoch": 0.15013698630136987,
+      "grad_norm": 3.1494054794311523,
+      "learning_rate": 2.244525547445256e-06,
+      "loss": 2.2866,
+      "step": 137
+    },
+    {
+      "epoch": 0.15123287671232877,
+      "grad_norm": 2.723390817642212,
+      "learning_rate": 2.2627737226277373e-06,
+      "loss": 2.3109,
+      "step": 138
+    },
+    {
+      "epoch": 0.15232876712328766,
+      "grad_norm": 2.8904967308044434,
+      "learning_rate": 2.281021897810219e-06,
+      "loss": 2.301,
+      "step": 139
+    },
+    {
+      "epoch": 0.15342465753424658,
+      "grad_norm": 2.8899483680725098,
+      "learning_rate": 2.299270072992701e-06,
+      "loss": 2.2637,
+      "step": 140
+    },
+    {
+      "epoch": 0.15342465753424658,
+      "eval_loss": 2.3952724933624268,
+      "eval_runtime": 37.1272,
+      "eval_samples_per_second": 26.934,
+      "eval_steps_per_second": 1.131,
+      "step": 140
+    },
+    {
+      "epoch": 0.15452054794520548,
+      "grad_norm": 3.5309066772460938,
+      "learning_rate": 2.3175182481751825e-06,
+      "loss": 2.2572,
+      "step": 141
+    },
+    {
+      "epoch": 0.15561643835616437,
+      "grad_norm": 2.8369667530059814,
+      "learning_rate": 2.3357664233576643e-06,
+      "loss": 2.2712,
+      "step": 142
+    },
+    {
+      "epoch": 0.1567123287671233,
+      "grad_norm": 3.629161834716797,
+      "learning_rate": 2.354014598540146e-06,
+      "loss": 2.2301,
+      "step": 143
+    },
+    {
+      "epoch": 0.1578082191780822,
+      "grad_norm": 3.5313000679016113,
+      "learning_rate": 2.3722627737226276e-06,
+      "loss": 2.2712,
+      "step": 144
+    },
+    {
+      "epoch": 0.1589041095890411,
+      "grad_norm": 3.1328771114349365,
+      "learning_rate": 2.39051094890511e-06,
+      "loss": 2.2706,
+      "step": 145
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.6992950439453125,
+      "learning_rate": 2.4087591240875918e-06,
+      "loss": 2.219,
+      "step": 146
+    },
+    {
+      "epoch": 0.1610958904109589,
+      "grad_norm": 5.230221748352051,
+      "learning_rate": 2.427007299270073e-06,
+      "loss": 2.2396,
+      "step": 147
+    },
+    {
+      "epoch": 0.16219178082191782,
+      "grad_norm": 3.5680761337280273,
+      "learning_rate": 2.445255474452555e-06,
+      "loss": 2.2434,
+      "step": 148
+    },
+    {
+      "epoch": 0.1632876712328767,
+      "grad_norm": 3.1642441749572754,
+      "learning_rate": 2.463503649635037e-06,
+      "loss": 2.2856,
+      "step": 149
+    },
+    {
+      "epoch": 0.1643835616438356,
+      "grad_norm": 2.694085121154785,
+      "learning_rate": 2.4817518248175183e-06,
+      "loss": 2.2006,
+      "step": 150
+    },
+    {
+      "epoch": 0.1643835616438356,
+      "eval_loss": 2.37199068069458,
+      "eval_runtime": 37.1342,
+      "eval_samples_per_second": 26.929,
+      "eval_steps_per_second": 1.131,
+      "step": 150
+    },
+    {
+      "epoch": 0.16547945205479453,
+      "grad_norm": 2.9868853092193604,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1694,
+      "step": 151
+    },
+    {
+      "epoch": 0.16657534246575342,
+      "grad_norm": 2.793532371520996,
+      "learning_rate": 2.518248175182482e-06,
+      "loss": 2.2965,
+      "step": 152
+    },
+    {
+      "epoch": 0.16767123287671232,
+      "grad_norm": 2.60152530670166,
+      "learning_rate": 2.5364963503649635e-06,
+      "loss": 2.1954,
+      "step": 153
+    },
+    {
+      "epoch": 0.16876712328767124,
+      "grad_norm": 2.6192219257354736,
+      "learning_rate": 2.5547445255474458e-06,
+      "loss": 2.1909,
+      "step": 154
+    },
+    {
+      "epoch": 0.16986301369863013,
+      "grad_norm": 2.467344045639038,
+      "learning_rate": 2.5729927007299272e-06,
+      "loss": 2.2438,
+      "step": 155
+    },
+    {
+      "epoch": 0.17095890410958905,
+      "grad_norm": 2.772012948989868,
+      "learning_rate": 2.591240875912409e-06,
+      "loss": 2.2213,
+      "step": 156
+    },
+    {
+      "epoch": 0.17205479452054795,
+      "grad_norm": 2.613675355911255,
+      "learning_rate": 2.6094890510948905e-06,
+      "loss": 2.2213,
+      "step": 157
+    },
+    {
+      "epoch": 0.17315068493150684,
+      "grad_norm": 2.849245071411133,
+      "learning_rate": 2.627737226277373e-06,
+      "loss": 2.255,
+      "step": 158
+    },
+    {
+      "epoch": 0.17424657534246576,
+      "grad_norm": 2.8388969898223877,
+      "learning_rate": 2.6459854014598542e-06,
+      "loss": 2.2572,
+      "step": 159
+    },
+    {
+      "epoch": 0.17534246575342466,
+      "grad_norm": 2.5219321250915527,
+      "learning_rate": 2.664233576642336e-06,
+      "loss": 2.2253,
+      "step": 160
+    },
+    {
+      "epoch": 0.17534246575342466,
+      "eval_loss": 2.3509061336517334,
+      "eval_runtime": 37.0953,
+      "eval_samples_per_second": 26.958,
+      "eval_steps_per_second": 1.132,
+      "step": 160
+    },
+    {
+      "epoch": 0.17643835616438355,
+      "grad_norm": 2.8460891246795654,
+      "learning_rate": 2.6824817518248175e-06,
+      "loss": 2.2009,
+      "step": 161
+    },
+    {
+      "epoch": 0.17753424657534247,
+      "grad_norm": 2.673377275466919,
+      "learning_rate": 2.7007299270072994e-06,
+      "loss": 2.2211,
+      "step": 162
+    },
+    {
+      "epoch": 0.17863013698630137,
+      "grad_norm": 2.8514182567596436,
+      "learning_rate": 2.718978102189781e-06,
+      "loss": 2.1955,
+      "step": 163
+    },
+    {
+      "epoch": 0.17972602739726026,
+      "grad_norm": 3.530385971069336,
+      "learning_rate": 2.737226277372263e-06,
+      "loss": 2.2536,
+      "step": 164
+    },
+    {
+      "epoch": 0.18082191780821918,
+      "grad_norm": 2.733394145965576,
+      "learning_rate": 2.755474452554745e-06,
+      "loss": 2.2367,
+      "step": 165
+    },
+    {
+      "epoch": 0.18191780821917808,
+      "grad_norm": 3.871150016784668,
+      "learning_rate": 2.7737226277372264e-06,
+      "loss": 2.1949,
+      "step": 166
+    },
+    {
+      "epoch": 0.18301369863013697,
+      "grad_norm": 3.7782530784606934,
+      "learning_rate": 2.7919708029197082e-06,
+      "loss": 2.2979,
+      "step": 167
+    },
+    {
+      "epoch": 0.1841095890410959,
+      "grad_norm": 3.094599485397339,
+      "learning_rate": 2.81021897810219e-06,
+      "loss": 2.2512,
+      "step": 168
+    },
+    {
+      "epoch": 0.1852054794520548,
+      "grad_norm": 3.2166848182678223,
+      "learning_rate": 2.828467153284672e-06,
+      "loss": 2.1784,
+      "step": 169
+    },
+    {
+      "epoch": 0.1863013698630137,
+      "grad_norm": 3.4127073287963867,
+      "learning_rate": 2.8467153284671534e-06,
+      "loss": 2.2241,
+      "step": 170
+    },
+    {
+      "epoch": 0.1863013698630137,
+      "eval_loss": 2.333245038986206,
+      "eval_runtime": 37.0402,
+      "eval_samples_per_second": 26.998,
+      "eval_steps_per_second": 1.134,
+      "step": 170
+    },
+    {
+      "epoch": 0.1873972602739726,
+      "grad_norm": 2.677579879760742,
+      "learning_rate": 2.8649635036496353e-06,
+      "loss": 2.135,
+      "step": 171
+    },
+    {
+      "epoch": 0.1884931506849315,
+      "grad_norm": 2.9602150917053223,
+      "learning_rate": 2.8832116788321167e-06,
+      "loss": 2.1916,
+      "step": 172
+    },
+    {
+      "epoch": 0.18958904109589042,
+      "grad_norm": 2.580122232437134,
+      "learning_rate": 2.901459854014599e-06,
+      "loss": 2.1477,
+      "step": 173
+    },
+    {
+      "epoch": 0.19068493150684931,
+      "grad_norm": 2.652217388153076,
+      "learning_rate": 2.9197080291970804e-06,
+      "loss": 2.2394,
+      "step": 174
+    },
+    {
+      "epoch": 0.1917808219178082,
+      "grad_norm": 2.6887567043304443,
+      "learning_rate": 2.9379562043795623e-06,
+      "loss": 2.2245,
+      "step": 175
+    },
+    {
+      "epoch": 0.19287671232876713,
+      "grad_norm": 2.768165111541748,
+      "learning_rate": 2.956204379562044e-06,
+      "loss": 2.2017,
+      "step": 176
+    },
+    {
+      "epoch": 0.19397260273972602,
+      "grad_norm": 2.9375691413879395,
+      "learning_rate": 2.9744525547445256e-06,
+      "loss": 2.2464,
+      "step": 177
+    },
+    {
+      "epoch": 0.19506849315068492,
+      "grad_norm": 2.8364248275756836,
+      "learning_rate": 2.992700729927008e-06,
+      "loss": 2.2155,
+      "step": 178
+    },
+    {
+      "epoch": 0.19616438356164384,
+      "grad_norm": 2.843487024307251,
+      "learning_rate": 3.0109489051094893e-06,
+      "loss": 2.1577,
+      "step": 179
+    },
+    {
+      "epoch": 0.19726027397260273,
+      "grad_norm": 3.005230665206909,
+      "learning_rate": 3.029197080291971e-06,
+      "loss": 2.1549,
+      "step": 180
+    },
+    {
+      "epoch": 0.19726027397260273,
+      "eval_loss": 2.3209409713745117,
+      "eval_runtime": 37.022,
+      "eval_samples_per_second": 27.011,
+      "eval_steps_per_second": 1.134,
+      "step": 180
+    },
+    {
+      "epoch": 0.19835616438356166,
+      "grad_norm": 2.490260601043701,
+      "learning_rate": 3.0474452554744526e-06,
+      "loss": 2.2313,
+      "step": 181
+    },
+    {
+      "epoch": 0.19945205479452055,
+      "grad_norm": 2.973118543624878,
+      "learning_rate": 3.065693430656935e-06,
+      "loss": 2.2316,
+      "step": 182
+    },
+    {
+      "epoch": 0.20054794520547944,
+      "grad_norm": 3.3308680057525635,
+      "learning_rate": 3.0839416058394163e-06,
+      "loss": 2.2087,
+      "step": 183
+    },
+    {
+      "epoch": 0.20164383561643837,
+      "grad_norm": 3.1440727710723877,
+      "learning_rate": 3.102189781021898e-06,
+      "loss": 2.2256,
+      "step": 184
+    },
+    {
+      "epoch": 0.20273972602739726,
+      "grad_norm": 2.5696325302124023,
+      "learning_rate": 3.1204379562043796e-06,
+      "loss": 2.2093,
+      "step": 185
+    },
+    {
+      "epoch": 0.20383561643835615,
+      "grad_norm": 3.1225037574768066,
+      "learning_rate": 3.1386861313868614e-06,
+      "loss": 2.1425,
+      "step": 186
+    },
+    {
+      "epoch": 0.20493150684931508,
+      "grad_norm": 3.650743246078491,
+      "learning_rate": 3.156934306569343e-06,
+      "loss": 2.1339,
+      "step": 187
+    },
+    {
+      "epoch": 0.20602739726027397,
+      "grad_norm": 2.924072027206421,
+      "learning_rate": 3.175182481751825e-06,
+      "loss": 2.2363,
+      "step": 188
+    },
+    {
+      "epoch": 0.20712328767123286,
+      "grad_norm": 2.9561989307403564,
+      "learning_rate": 3.193430656934307e-06,
+      "loss": 2.1934,
+      "step": 189
+    },
+    {
+      "epoch": 0.20821917808219179,
+      "grad_norm": 2.731393814086914,
+      "learning_rate": 3.2116788321167884e-06,
+      "loss": 2.1718,
+      "step": 190
+    },
+    {
+      "epoch": 0.20821917808219179,
+      "eval_loss": 2.3087782859802246,
+      "eval_runtime": 37.0161,
+      "eval_samples_per_second": 27.015,
+      "eval_steps_per_second": 1.135,
+      "step": 190
+    },
+    {
+      "epoch": 0.20931506849315068,
+      "grad_norm": 2.774442434310913,
+      "learning_rate": 3.2299270072992707e-06,
+      "loss": 2.1841,
+      "step": 191
+    },
+    {
+      "epoch": 0.2104109589041096,
+      "grad_norm": 3.0579135417938232,
+      "learning_rate": 3.248175182481752e-06,
+      "loss": 2.1677,
+      "step": 192
+    },
+    {
+      "epoch": 0.2115068493150685,
+      "grad_norm": 2.732984781265259,
+      "learning_rate": 3.266423357664234e-06,
+      "loss": 2.0975,
+      "step": 193
+    },
+    {
+      "epoch": 0.2126027397260274,
+      "grad_norm": 3.442659854888916,
+      "learning_rate": 3.2846715328467155e-06,
+      "loss": 2.1923,
+      "step": 194
+    },
+    {
+      "epoch": 0.2136986301369863,
+      "grad_norm": 3.064223527908325,
+      "learning_rate": 3.3029197080291973e-06,
+      "loss": 2.1707,
+      "step": 195
+    },
+    {
+      "epoch": 0.2147945205479452,
+      "grad_norm": 2.5138468742370605,
+      "learning_rate": 3.3211678832116788e-06,
+      "loss": 2.1933,
+      "step": 196
+    },
+    {
+      "epoch": 0.2158904109589041,
+      "grad_norm": 3.1076464653015137,
+      "learning_rate": 3.339416058394161e-06,
+      "loss": 2.1725,
+      "step": 197
+    },
+    {
+      "epoch": 0.21698630136986302,
+      "grad_norm": 2.7291388511657715,
+      "learning_rate": 3.3576642335766425e-06,
+      "loss": 2.1722,
+      "step": 198
+    },
+    {
+      "epoch": 0.21808219178082192,
+      "grad_norm": 2.805506467819214,
+      "learning_rate": 3.3759124087591243e-06,
+      "loss": 2.1688,
+      "step": 199
+    },
+    {
+      "epoch": 0.2191780821917808,
+      "grad_norm": 2.6190085411071777,
+      "learning_rate": 3.3941605839416058e-06,
+      "loss": 2.1708,
+      "step": 200
+    },
+    {
+      "epoch": 0.2191780821917808,
+      "eval_loss": 2.295546054840088,
+      "eval_runtime": 37.0133,
+      "eval_samples_per_second": 27.017,
+      "eval_steps_per_second": 1.135,
+      "step": 200
+    },
+    {
+      "epoch": 0.22027397260273973,
+      "grad_norm": 3.1270673274993896,
+      "learning_rate": 3.412408759124088e-06,
+      "loss": 2.1759,
+      "step": 201
+    },
+    {
+      "epoch": 0.22136986301369863,
+      "grad_norm": 3.494178533554077,
+      "learning_rate": 3.43065693430657e-06,
+      "loss": 2.1795,
+      "step": 202
+    },
+    {
+      "epoch": 0.22246575342465755,
+      "grad_norm": 2.7398736476898193,
+      "learning_rate": 3.4489051094890513e-06,
+      "loss": 2.1201,
+      "step": 203
+    },
+    {
+      "epoch": 0.22356164383561644,
+      "grad_norm": 2.981506824493408,
+      "learning_rate": 3.467153284671533e-06,
+      "loss": 2.1693,
+      "step": 204
+    },
+    {
+      "epoch": 0.22465753424657534,
+      "grad_norm": 2.4129178524017334,
+      "learning_rate": 3.4854014598540146e-06,
+      "loss": 2.1125,
+      "step": 205
+    },
+    {
+      "epoch": 0.22575342465753426,
+      "grad_norm": 2.701197385787964,
+      "learning_rate": 3.503649635036497e-06,
+      "loss": 2.1565,
+      "step": 206
+    },
+    {
+      "epoch": 0.22684931506849315,
+      "grad_norm": 2.533306837081909,
+      "learning_rate": 3.5218978102189783e-06,
+      "loss": 2.1487,
+      "step": 207
+    },
+    {
+      "epoch": 0.22794520547945205,
+      "grad_norm": 2.8030788898468018,
+      "learning_rate": 3.54014598540146e-06,
+      "loss": 2.1801,
+      "step": 208
+    },
+    {
+      "epoch": 0.22904109589041097,
+      "grad_norm": 2.895414352416992,
+      "learning_rate": 3.5583941605839416e-06,
+      "loss": 2.189,
+      "step": 209
+    },
+    {
+      "epoch": 0.23013698630136986,
+      "grad_norm": 2.508390426635742,
+      "learning_rate": 3.576642335766424e-06,
+      "loss": 2.2376,
+      "step": 210
+    },
+    {
+      "epoch": 0.23013698630136986,
+      "eval_loss": 2.2900516986846924,
+      "eval_runtime": 36.7645,
+      "eval_samples_per_second": 27.2,
+      "eval_steps_per_second": 1.142,
+      "step": 210
+    },
+    {
+      "epoch": 0.23123287671232876,
+      "grad_norm": 2.6393816471099854,
+      "learning_rate": 3.5948905109489054e-06,
+      "loss": 2.2267,
+      "step": 211
+    },
+    {
+      "epoch": 0.23232876712328768,
+      "grad_norm": 3.0687549114227295,
+      "learning_rate": 3.6131386861313872e-06,
+      "loss": 2.0744,
+      "step": 212
+    },
+    {
+      "epoch": 0.23342465753424657,
+      "grad_norm": 2.6342105865478516,
+      "learning_rate": 3.6313868613138687e-06,
+      "loss": 2.0737,
+      "step": 213
+    },
+    {
+      "epoch": 0.2345205479452055,
+      "grad_norm": 3.5662875175476074,
+      "learning_rate": 3.6496350364963505e-06,
+      "loss": 2.1147,
+      "step": 214
+    },
+    {
+      "epoch": 0.2356164383561644,
+      "grad_norm": 2.5357561111450195,
+      "learning_rate": 3.667883211678833e-06,
+      "loss": 2.1268,
+      "step": 215
+    },
+    {
+      "epoch": 0.23671232876712328,
+      "grad_norm": 3.439711093902588,
+      "learning_rate": 3.6861313868613142e-06,
+      "loss": 2.1118,
+      "step": 216
+    },
+    {
+      "epoch": 0.2378082191780822,
+      "grad_norm": 2.9410204887390137,
+      "learning_rate": 3.704379562043796e-06,
+      "loss": 2.1087,
+      "step": 217
+    },
+    {
+      "epoch": 0.2389041095890411,
+      "grad_norm": 3.3182294368743896,
+      "learning_rate": 3.7226277372262775e-06,
+      "loss": 2.196,
+      "step": 218
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 3.3834011554718018,
+      "learning_rate": 3.7408759124087594e-06,
+      "loss": 2.1633,
+      "step": 219
+    },
+    {
+      "epoch": 0.2410958904109589,
+      "grad_norm": 3.114952325820923,
+      "learning_rate": 3.7591240875912412e-06,
+      "loss": 2.1107,
+      "step": 220
+    },
+    {
+      "epoch": 0.2410958904109589,
+      "eval_loss": 2.277580976486206,
+      "eval_runtime": 36.9846,
+      "eval_samples_per_second": 27.038,
+      "eval_steps_per_second": 1.136,
+      "step": 220
+    },
+    {
+      "epoch": 0.2421917808219178,
+      "grad_norm": 3.1151392459869385,
+      "learning_rate": 3.777372262773723e-06,
+      "loss": 2.1471,
+      "step": 221
+    },
+    {
+      "epoch": 0.2432876712328767,
+      "grad_norm": 3.798450469970703,
+      "learning_rate": 3.7956204379562045e-06,
+      "loss": 2.1324,
+      "step": 222
+    },
+    {
+      "epoch": 0.24438356164383562,
+      "grad_norm": 2.6187098026275635,
+      "learning_rate": 3.8138686131386864e-06,
+      "loss": 2.1061,
+      "step": 223
+    },
+    {
+      "epoch": 0.24547945205479452,
+      "grad_norm": 4.014018535614014,
+      "learning_rate": 3.832116788321168e-06,
+      "loss": 2.166,
+      "step": 224
+    },
+    {
+      "epoch": 0.2465753424657534,
+      "grad_norm": 3.0043938159942627,
+      "learning_rate": 3.85036496350365e-06,
+      "loss": 2.1197,
+      "step": 225
+    },
+    {
+      "epoch": 0.24767123287671233,
+      "grad_norm": 3.507066249847412,
+      "learning_rate": 3.868613138686132e-06,
+      "loss": 2.0848,
+      "step": 226
+    },
+    {
+      "epoch": 0.24876712328767123,
+      "grad_norm": 3.5713281631469727,
+      "learning_rate": 3.886861313868613e-06,
+      "loss": 2.1239,
+      "step": 227
+    },
+    {
+      "epoch": 0.24986301369863015,
+      "grad_norm": 3.961082696914673,
+      "learning_rate": 3.905109489051096e-06,
+      "loss": 2.1406,
+      "step": 228
+    },
+    {
+      "epoch": 0.25095890410958904,
+      "grad_norm": 3.1514859199523926,
+      "learning_rate": 3.923357664233577e-06,
+      "loss": 2.1689,
+      "step": 229
+    },
+    {
+      "epoch": 0.25205479452054796,
+      "grad_norm": 3.8486833572387695,
+      "learning_rate": 3.9416058394160585e-06,
+      "loss": 2.1321,
+      "step": 230
+    },
+    {
+      "epoch": 0.25205479452054796,
+      "eval_loss": 2.2620601654052734,
+      "eval_runtime": 37.058,
+      "eval_samples_per_second": 26.985,
+      "eval_steps_per_second": 1.133,
+      "step": 230
+    },
+    {
+      "epoch": 0.25315068493150683,
+      "grad_norm": 2.5626907348632812,
+      "learning_rate": 3.95985401459854e-06,
+      "loss": 2.1777,
+      "step": 231
+    },
+    {
+      "epoch": 0.25424657534246575,
+      "grad_norm": 4.105278015136719,
+      "learning_rate": 3.978102189781022e-06,
+      "loss": 2.2099,
+      "step": 232
+    },
+    {
+      "epoch": 0.2553424657534247,
+      "grad_norm": 2.9818739891052246,
+      "learning_rate": 3.996350364963504e-06,
+      "loss": 2.0705,
+      "step": 233
+    },
+    {
+      "epoch": 0.25643835616438354,
+      "grad_norm": 2.87030291557312,
+      "learning_rate": 4.014598540145986e-06,
+      "loss": 2.1453,
+      "step": 234
+    },
+    {
+      "epoch": 0.25753424657534246,
+      "grad_norm": 2.4847915172576904,
+      "learning_rate": 4.032846715328467e-06,
+      "loss": 2.0461,
+      "step": 235
+    },
+    {
+      "epoch": 0.2586301369863014,
+      "grad_norm": 3.451420783996582,
+      "learning_rate": 4.05109489051095e-06,
+      "loss": 2.1133,
+      "step": 236
+    },
+    {
+      "epoch": 0.25972602739726025,
+      "grad_norm": 2.954723358154297,
+      "learning_rate": 4.069343065693431e-06,
+      "loss": 2.1316,
+      "step": 237
+    },
+    {
+      "epoch": 0.2608219178082192,
+      "grad_norm": 2.5295896530151367,
+      "learning_rate": 4.0875912408759126e-06,
+      "loss": 2.0901,
+      "step": 238
+    },
+    {
+      "epoch": 0.2619178082191781,
+      "grad_norm": 2.6359453201293945,
+      "learning_rate": 4.105839416058395e-06,
+      "loss": 2.117,
+      "step": 239
+    },
+    {
+      "epoch": 0.26301369863013696,
+      "grad_norm": 2.5236105918884277,
+      "learning_rate": 4.124087591240876e-06,
+      "loss": 2.1958,
+      "step": 240
+    },
+    {
+      "epoch": 0.26301369863013696,
+      "eval_loss": 2.250295400619507,
+      "eval_runtime": 37.0811,
+      "eval_samples_per_second": 26.968,
+      "eval_steps_per_second": 1.133,
+      "step": 240
+    },
+    {
+      "epoch": 0.2641095890410959,
+      "grad_norm": 2.581918954849243,
+      "learning_rate": 4.1423357664233586e-06,
+      "loss": 2.1054,
+      "step": 241
+    },
+    {
+      "epoch": 0.2652054794520548,
+      "grad_norm": 2.501332998275757,
+      "learning_rate": 4.16058394160584e-06,
+      "loss": 2.0878,
+      "step": 242
+    },
+    {
+      "epoch": 0.2663013698630137,
+      "grad_norm": 2.6477770805358887,
+      "learning_rate": 4.1788321167883214e-06,
+      "loss": 2.1656,
+      "step": 243
+    },
+    {
+      "epoch": 0.2673972602739726,
+      "grad_norm": 2.7252438068389893,
+      "learning_rate": 4.197080291970803e-06,
+      "loss": 2.0497,
+      "step": 244
+    },
+    {
+      "epoch": 0.2684931506849315,
+      "grad_norm": 2.6056301593780518,
+      "learning_rate": 4.215328467153285e-06,
+      "loss": 2.1411,
+      "step": 245
+    },
+    {
+      "epoch": 0.26958904109589044,
+      "grad_norm": 2.6581931114196777,
+      "learning_rate": 4.233576642335767e-06,
+      "loss": 2.0829,
+      "step": 246
+    },
+    {
+      "epoch": 0.2706849315068493,
+      "grad_norm": 2.6316752433776855,
+      "learning_rate": 4.251824817518249e-06,
+      "loss": 2.0882,
+      "step": 247
+    },
+    {
+      "epoch": 0.2717808219178082,
+      "grad_norm": 2.803823947906494,
+      "learning_rate": 4.27007299270073e-06,
+      "loss": 2.1203,
+      "step": 248
+    },
+    {
+      "epoch": 0.27287671232876715,
+      "grad_norm": 2.9551174640655518,
+      "learning_rate": 4.288321167883212e-06,
+      "loss": 2.0897,
+      "step": 249
+    },
+    {
+      "epoch": 0.273972602739726,
+      "grad_norm": 2.4847161769866943,
+      "learning_rate": 4.306569343065693e-06,
+      "loss": 2.0845,
+      "step": 250
+    },
+    {
+      "epoch": 0.273972602739726,
+      "eval_loss": 2.2491259574890137,
+      "eval_runtime": 37.1024,
+      "eval_samples_per_second": 26.952,
+      "eval_steps_per_second": 1.132,
+      "step": 250
+    },
+    {
+      "epoch": 0.27506849315068493,
+      "grad_norm": 2.6398351192474365,
+      "learning_rate": 4.3248175182481755e-06,
+      "loss": 2.0479,
+      "step": 251
+    },
+    {
+      "epoch": 0.27616438356164386,
+      "grad_norm": 3.0948562622070312,
+      "learning_rate": 4.343065693430658e-06,
+      "loss": 2.1184,
+      "step": 252
+    },
+    {
+      "epoch": 0.2772602739726027,
+      "grad_norm": 2.5378012657165527,
+      "learning_rate": 4.361313868613139e-06,
+      "loss": 2.1099,
+      "step": 253
+    },
+    {
+      "epoch": 0.27835616438356164,
+      "grad_norm": 3.0282225608825684,
+      "learning_rate": 4.379562043795621e-06,
+      "loss": 2.0982,
+      "step": 254
+    },
+    {
+      "epoch": 0.27945205479452057,
+      "grad_norm": 2.6205334663391113,
+      "learning_rate": 4.397810218978102e-06,
+      "loss": 2.1285,
+      "step": 255
+    },
+    {
+      "epoch": 0.28054794520547943,
+      "grad_norm": 3.2056095600128174,
+      "learning_rate": 4.416058394160584e-06,
+      "loss": 2.1221,
+      "step": 256
+    },
+    {
+      "epoch": 0.28164383561643835,
+      "grad_norm": 3.1370127201080322,
+      "learning_rate": 4.434306569343066e-06,
+      "loss": 2.1016,
+      "step": 257
+    },
+    {
+      "epoch": 0.2827397260273973,
+      "grad_norm": 2.494168519973755,
+      "learning_rate": 4.452554744525548e-06,
+      "loss": 2.0754,
+      "step": 258
+    },
+    {
+      "epoch": 0.28383561643835614,
+      "grad_norm": 2.7979183197021484,
+      "learning_rate": 4.4708029197080295e-06,
+      "loss": 2.1218,
+      "step": 259
+    },
+    {
+      "epoch": 0.28493150684931506,
+      "grad_norm": 2.755682945251465,
+      "learning_rate": 4.489051094890512e-06,
+      "loss": 2.1136,
+      "step": 260
+    },
+    {
+      "epoch": 0.28493150684931506,
+      "eval_loss": 2.2433817386627197,
+      "eval_runtime": 37.1121,
+      "eval_samples_per_second": 26.945,
+      "eval_steps_per_second": 1.132,
+      "step": 260
+    },
+    {
+      "epoch": 0.286027397260274,
+      "grad_norm": 2.3641891479492188,
+      "learning_rate": 4.507299270072993e-06,
+      "loss": 2.1488,
+      "step": 261
+    },
+    {
+      "epoch": 0.28712328767123285,
+      "grad_norm": 2.741292953491211,
+      "learning_rate": 4.525547445255475e-06,
+      "loss": 2.1048,
+      "step": 262
+    },
+    {
+      "epoch": 0.2882191780821918,
+      "grad_norm": 2.811483383178711,
+      "learning_rate": 4.543795620437956e-06,
+      "loss": 2.0996,
+      "step": 263
+    },
+    {
+      "epoch": 0.2893150684931507,
+      "grad_norm": 2.6705479621887207,
+      "learning_rate": 4.562043795620438e-06,
+      "loss": 2.0652,
+      "step": 264
+    },
+    {
+      "epoch": 0.29041095890410956,
+      "grad_norm": 2.9383747577667236,
+      "learning_rate": 4.580291970802921e-06,
+      "loss": 2.1574,
+      "step": 265
+    },
+    {
+      "epoch": 0.2915068493150685,
+      "grad_norm": 2.591373920440674,
+      "learning_rate": 4.598540145985402e-06,
+      "loss": 2.1655,
+      "step": 266
+    },
+    {
+      "epoch": 0.2926027397260274,
+      "grad_norm": 3.16955828666687,
+      "learning_rate": 4.6167883211678835e-06,
+      "loss": 2.1285,
+      "step": 267
+    },
+    {
+      "epoch": 0.2936986301369863,
+      "grad_norm": 2.66941499710083,
+      "learning_rate": 4.635036496350365e-06,
+      "loss": 2.0777,
+      "step": 268
+    },
+    {
+      "epoch": 0.2947945205479452,
+      "grad_norm": 2.768139600753784,
+      "learning_rate": 4.653284671532847e-06,
+      "loss": 2.0748,
+      "step": 269
+    },
+    {
+      "epoch": 0.2958904109589041,
+      "grad_norm": 2.5951924324035645,
+      "learning_rate": 4.671532846715329e-06,
+      "loss": 2.1289,
+      "step": 270
+    },
+    {
+      "epoch": 0.2958904109589041,
+      "eval_loss": 2.2393550872802734,
+      "eval_runtime": 37.0993,
+      "eval_samples_per_second": 26.955,
+      "eval_steps_per_second": 1.132,
+      "step": 270
+    },
+    {
+      "epoch": 0.29698630136986304,
+      "grad_norm": 2.5422160625457764,
+      "learning_rate": 4.689781021897811e-06,
+      "loss": 2.1342,
+      "step": 271
+    },
+    {
+      "epoch": 0.2980821917808219,
+      "grad_norm": 2.540555477142334,
+      "learning_rate": 4.708029197080292e-06,
+      "loss": 2.097,
+      "step": 272
+    },
+    {
+      "epoch": 0.2991780821917808,
+      "grad_norm": 2.995366334915161,
+      "learning_rate": 4.726277372262774e-06,
+      "loss": 2.1452,
+      "step": 273
+    },
+    {
+      "epoch": 0.30027397260273975,
+      "grad_norm": 2.2561020851135254,
+      "learning_rate": 4.744525547445255e-06,
+      "loss": 2.1069,
+      "step": 274
+    },
+    {
+      "epoch": 0.3013698630136986,
+      "grad_norm": 2.53476619720459,
+      "learning_rate": 4.7627737226277375e-06,
+      "loss": 2.0381,
+      "step": 275
+    },
+    {
+      "epoch": 0.30246575342465754,
+      "grad_norm": 2.3415749073028564,
+      "learning_rate": 4.78102189781022e-06,
+      "loss": 2.1027,
+      "step": 276
+    },
+    {
+      "epoch": 0.30356164383561646,
+      "grad_norm": 2.9756431579589844,
+      "learning_rate": 4.799270072992701e-06,
+      "loss": 2.1136,
+      "step": 277
+    },
+    {
+      "epoch": 0.3046575342465753,
+      "grad_norm": 2.606208324432373,
+      "learning_rate": 4.8175182481751835e-06,
+      "loss": 2.0797,
+      "step": 278
+    },
+    {
+      "epoch": 0.30575342465753425,
+      "grad_norm": 2.5089313983917236,
+      "learning_rate": 4.835766423357665e-06,
+      "loss": 2.1026,
+      "step": 279
+    },
+    {
+      "epoch": 0.30684931506849317,
+      "grad_norm": 2.5003838539123535,
+      "learning_rate": 4.854014598540146e-06,
+      "loss": 2.0923,
+      "step": 280
+    },
+    {
+      "epoch": 0.30684931506849317,
+      "eval_loss": 2.2351036071777344,
+      "eval_runtime": 37.111,
+      "eval_samples_per_second": 26.946,
+      "eval_steps_per_second": 1.132,
+      "step": 280
+    },
+    {
+      "epoch": 0.30794520547945203,
+      "grad_norm": 2.403602361679077,
+      "learning_rate": 4.872262773722628e-06,
+      "loss": 2.0388,
+      "step": 281
+    },
+    {
+      "epoch": 0.30904109589041096,
+      "grad_norm": 2.7775959968566895,
+      "learning_rate": 4.89051094890511e-06,
+      "loss": 2.0561,
+      "step": 282
+    },
+    {
+      "epoch": 0.3101369863013699,
+      "grad_norm": 2.7128653526306152,
+      "learning_rate": 4.9087591240875915e-06,
+      "loss": 2.042,
+      "step": 283
+    },
+    {
+      "epoch": 0.31123287671232874,
+      "grad_norm": 2.7029991149902344,
+      "learning_rate": 4.927007299270074e-06,
+      "loss": 2.0814,
+      "step": 284
+    },
+    {
+      "epoch": 0.31232876712328766,
+      "grad_norm": 2.3942933082580566,
+      "learning_rate": 4.945255474452555e-06,
+      "loss": 2.0906,
+      "step": 285
+    },
+    {
+      "epoch": 0.3134246575342466,
+      "grad_norm": 2.5086541175842285,
+      "learning_rate": 4.963503649635037e-06,
+      "loss": 2.0625,
+      "step": 286
+    },
+    {
+      "epoch": 0.31452054794520545,
+      "grad_norm": 2.6668760776519775,
+      "learning_rate": 4.981751824817518e-06,
+      "loss": 2.1538,
+      "step": 287
+    },
+    {
+      "epoch": 0.3156164383561644,
+      "grad_norm": 2.5066750049591064,
+      "learning_rate": 5e-06,
+      "loss": 2.1423,
+      "step": 288
+    },
+    {
+      "epoch": 0.3167123287671233,
+      "grad_norm": 2.84967041015625,
+      "learning_rate": 4.997969130787978e-06,
+      "loss": 2.0943,
+      "step": 289
+    },
+    {
+      "epoch": 0.3178082191780822,
+      "grad_norm": 2.151946544647217,
+      "learning_rate": 4.995938261575955e-06,
+      "loss": 2.1098,
+      "step": 290
+    },
+    {
+      "epoch": 0.3178082191780822,
+      "eval_loss": 2.223247528076172,
+      "eval_runtime": 37.1755,
+      "eval_samples_per_second": 26.899,
+      "eval_steps_per_second": 1.13,
+      "step": 290
+    },
+    {
+      "epoch": 0.3189041095890411,
+      "grad_norm": 2.887166976928711,
+      "learning_rate": 4.993907392363932e-06,
+      "loss": 2.0281,
+      "step": 291
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 2.5930211544036865,
+      "learning_rate": 4.9918765231519095e-06,
+      "loss": 2.1063,
+      "step": 292
+    },
+    {
+      "epoch": 0.32109589041095893,
+      "grad_norm": 2.5568413734436035,
+      "learning_rate": 4.989845653939887e-06,
+      "loss": 2.0313,
+      "step": 293
+    },
+    {
+      "epoch": 0.3221917808219178,
+      "grad_norm": 2.536095380783081,
+      "learning_rate": 4.987814784727864e-06,
+      "loss": 2.0749,
+      "step": 294
+    },
+    {
+      "epoch": 0.3232876712328767,
+      "grad_norm": 2.3385651111602783,
+      "learning_rate": 4.985783915515841e-06,
+      "loss": 2.0666,
+      "step": 295
+    },
+    {
+      "epoch": 0.32438356164383564,
+      "grad_norm": 2.504826068878174,
+      "learning_rate": 4.983753046303819e-06,
+      "loss": 2.0677,
+      "step": 296
+    },
+    {
+      "epoch": 0.3254794520547945,
+      "grad_norm": 2.5289409160614014,
+      "learning_rate": 4.981722177091796e-06,
+      "loss": 2.0828,
+      "step": 297
+    },
+    {
+      "epoch": 0.3265753424657534,
+      "grad_norm": 2.803795099258423,
+      "learning_rate": 4.979691307879773e-06,
+      "loss": 2.0733,
+      "step": 298
+    },
+    {
+      "epoch": 0.32767123287671235,
+      "grad_norm": 2.651273012161255,
+      "learning_rate": 4.9776604386677504e-06,
+      "loss": 2.0987,
+      "step": 299
+    },
+    {
+      "epoch": 0.3287671232876712,
+      "grad_norm": 2.4715864658355713,
+      "learning_rate": 4.975629569455728e-06,
+      "loss": 2.0733,
+      "step": 300
+    },
+    {
+      "epoch": 0.3287671232876712,
+      "eval_loss": 2.214858293533325,
+      "eval_runtime": 37.0559,
+      "eval_samples_per_second": 26.986,
+      "eval_steps_per_second": 1.133,
+      "step": 300
+    },
+    {
+      "epoch": 0.32986301369863014,
+      "grad_norm": 2.2573442459106445,
+      "learning_rate": 4.973598700243705e-06,
+      "loss": 2.1291,
+      "step": 301
+    },
+    {
+      "epoch": 0.33095890410958906,
+      "grad_norm": 2.645966053009033,
+      "learning_rate": 4.971567831031681e-06,
+      "loss": 2.046,
+      "step": 302
+    },
+    {
+      "epoch": 0.3320547945205479,
+      "grad_norm": 2.361392021179199,
+      "learning_rate": 4.9695369618196595e-06,
+      "loss": 2.0447,
+      "step": 303
+    },
+    {
+      "epoch": 0.33315068493150685,
+      "grad_norm": 2.272714376449585,
+      "learning_rate": 4.967506092607636e-06,
+      "loss": 1.9272,
+      "step": 304
+    },
+    {
+      "epoch": 0.33424657534246577,
+      "grad_norm": 2.5846943855285645,
+      "learning_rate": 4.965475223395614e-06,
+      "loss": 2.0199,
+      "step": 305
+    },
+    {
+      "epoch": 0.33534246575342463,
+      "grad_norm": 2.38985013961792,
+      "learning_rate": 4.9634443541835905e-06,
+      "loss": 2.0394,
+      "step": 306
+    },
+    {
+      "epoch": 0.33643835616438356,
+      "grad_norm": 2.3526251316070557,
+      "learning_rate": 4.961413484971569e-06,
+      "loss": 2.0403,
+      "step": 307
+    },
+    {
+      "epoch": 0.3375342465753425,
+      "grad_norm": 2.399531602859497,
+      "learning_rate": 4.959382615759545e-06,
+      "loss": 2.0607,
+      "step": 308
+    },
+    {
+      "epoch": 0.33863013698630134,
+      "grad_norm": 2.50803804397583,
+      "learning_rate": 4.957351746547523e-06,
+      "loss": 2.0688,
+      "step": 309
+    },
+    {
+      "epoch": 0.33972602739726027,
+      "grad_norm": 2.278088092803955,
+      "learning_rate": 4.9553208773355e-06,
+      "loss": 2.0453,
+      "step": 310
+    },
+    {
+      "epoch": 0.33972602739726027,
+      "eval_loss": 2.2185235023498535,
+      "eval_runtime": 36.9712,
+      "eval_samples_per_second": 27.048,
+      "eval_steps_per_second": 1.136,
+      "step": 310
+    },
+    {
+      "epoch": 0.3408219178082192,
+      "grad_norm": 2.555300712585449,
+      "learning_rate": 4.953290008123478e-06,
+      "loss": 2.1726,
+      "step": 311
+    },
+    {
+      "epoch": 0.3419178082191781,
+      "grad_norm": 2.614870548248291,
+      "learning_rate": 4.951259138911454e-06,
+      "loss": 2.0659,
+      "step": 312
+    },
+    {
+      "epoch": 0.343013698630137,
+      "grad_norm": 2.1932661533355713,
+      "learning_rate": 4.9492282696994315e-06,
+      "loss": 1.9981,
+      "step": 313
+    },
+    {
+      "epoch": 0.3441095890410959,
+      "grad_norm": 2.707451581954956,
+      "learning_rate": 4.947197400487409e-06,
+      "loss": 2.1321,
+      "step": 314
+    },
+    {
+      "epoch": 0.3452054794520548,
+      "grad_norm": 2.4317946434020996,
+      "learning_rate": 4.945166531275386e-06,
+      "loss": 2.0548,
+      "step": 315
+    },
+    {
+      "epoch": 0.3463013698630137,
+      "grad_norm": 2.639883279800415,
+      "learning_rate": 4.943135662063363e-06,
+      "loss": 2.0512,
+      "step": 316
+    },
+    {
+      "epoch": 0.3473972602739726,
+      "grad_norm": 3.1968822479248047,
+      "learning_rate": 4.9411047928513406e-06,
+      "loss": 2.0611,
+      "step": 317
+    },
+    {
+      "epoch": 0.34849315068493153,
+      "grad_norm": 2.4361510276794434,
+      "learning_rate": 4.939073923639318e-06,
+      "loss": 2.0974,
+      "step": 318
+    },
+    {
+      "epoch": 0.3495890410958904,
+      "grad_norm": 2.8278262615203857,
+      "learning_rate": 4.937043054427295e-06,
+      "loss": 2.0992,
+      "step": 319
+    },
+    {
+      "epoch": 0.3506849315068493,
+      "grad_norm": 3.01247239112854,
+      "learning_rate": 4.935012185215272e-06,
+      "loss": 2.0372,
+      "step": 320
+    },
+    {
+      "epoch": 0.3506849315068493,
+      "eval_loss": 2.206442356109619,
+      "eval_runtime": 36.8797,
+      "eval_samples_per_second": 27.115,
+      "eval_steps_per_second": 1.139,
+      "step": 320
+    },
+    {
+      "epoch": 0.35178082191780824,
+      "grad_norm": 2.3372156620025635,
+      "learning_rate": 4.93298131600325e-06,
+      "loss": 2.0211,
+      "step": 321
+    },
+    {
+      "epoch": 0.3528767123287671,
+      "grad_norm": 2.91377329826355,
+      "learning_rate": 4.930950446791227e-06,
+      "loss": 2.0879,
+      "step": 322
+    },
+    {
+      "epoch": 0.353972602739726,
+      "grad_norm": 2.3207666873931885,
+      "learning_rate": 4.928919577579204e-06,
+      "loss": 2.056,
+      "step": 323
+    },
+    {
+      "epoch": 0.35506849315068495,
+      "grad_norm": 2.3260514736175537,
+      "learning_rate": 4.9268887083671815e-06,
+      "loss": 2.0684,
+      "step": 324
+    },
+    {
+      "epoch": 0.3561643835616438,
+      "grad_norm": 2.3070554733276367,
+      "learning_rate": 4.924857839155159e-06,
+      "loss": 2.0494,
+      "step": 325
+    },
+    {
+      "epoch": 0.35726027397260274,
+      "grad_norm": 2.189871072769165,
+      "learning_rate": 4.922826969943136e-06,
+      "loss": 2.0531,
+      "step": 326
+    },
+    {
+      "epoch": 0.35835616438356166,
+      "grad_norm": 2.553520917892456,
+      "learning_rate": 4.920796100731113e-06,
+      "loss": 2.0749,
+      "step": 327
+    },
+    {
+      "epoch": 0.3594520547945205,
+      "grad_norm": 2.62296199798584,
+      "learning_rate": 4.918765231519091e-06,
+      "loss": 1.9983,
+      "step": 328
+    },
+    {
+      "epoch": 0.36054794520547945,
+      "grad_norm": 2.4612369537353516,
+      "learning_rate": 4.916734362307068e-06,
+      "loss": 2.0641,
+      "step": 329
+    },
+    {
+      "epoch": 0.36164383561643837,
+      "grad_norm": 2.656157970428467,
+      "learning_rate": 4.914703493095045e-06,
+      "loss": 2.0148,
+      "step": 330
+    },
+    {
+      "epoch": 0.36164383561643837,
+      "eval_loss": 2.2054214477539062,
+      "eval_runtime": 36.886,
+      "eval_samples_per_second": 27.111,
+      "eval_steps_per_second": 1.139,
+      "step": 330
+    },
+    {
+      "epoch": 0.36273972602739724,
+      "grad_norm": 2.2823736667633057,
+      "learning_rate": 4.912672623883022e-06,
+      "loss": 2.0221,
+      "step": 331
+    },
+    {
+      "epoch": 0.36383561643835616,
+      "grad_norm": 2.8403289318084717,
+      "learning_rate": 4.910641754671e-06,
+      "loss": 2.0149,
+      "step": 332
+    },
+    {
+      "epoch": 0.3649315068493151,
+      "grad_norm": 2.453887462615967,
+      "learning_rate": 4.908610885458977e-06,
+      "loss": 2.0411,
+      "step": 333
+    },
+    {
+      "epoch": 0.36602739726027395,
+      "grad_norm": 2.6094300746917725,
+      "learning_rate": 4.906580016246954e-06,
+      "loss": 2.0293,
+      "step": 334
+    },
+    {
+      "epoch": 0.36712328767123287,
+      "grad_norm": 2.4412384033203125,
+      "learning_rate": 4.9045491470349315e-06,
+      "loss": 2.0778,
+      "step": 335
+    },
+    {
+      "epoch": 0.3682191780821918,
+      "grad_norm": 2.4435057640075684,
+      "learning_rate": 4.902518277822909e-06,
+      "loss": 2.0597,
+      "step": 336
+    },
+    {
+      "epoch": 0.3693150684931507,
+      "grad_norm": 2.2390244007110596,
+      "learning_rate": 4.900487408610886e-06,
+      "loss": 2.0654,
+      "step": 337
+    },
+    {
+      "epoch": 0.3704109589041096,
+      "grad_norm": 2.4243953227996826,
+      "learning_rate": 4.898456539398863e-06,
+      "loss": 2.0696,
+      "step": 338
+    },
+    {
+      "epoch": 0.3715068493150685,
+      "grad_norm": 2.375720262527466,
+      "learning_rate": 4.89642567018684e-06,
+      "loss": 2.0866,
+      "step": 339
+    },
+    {
+      "epoch": 0.3726027397260274,
+      "grad_norm": 2.6379668712615967,
+      "learning_rate": 4.894394800974818e-06,
+      "loss": 2.1552,
+      "step": 340
+    },
+    {
+      "epoch": 0.3726027397260274,
+      "eval_loss": 2.196237802505493,
+      "eval_runtime": 36.8953,
+      "eval_samples_per_second": 27.104,
+      "eval_steps_per_second": 1.138,
+      "step": 340
+    },
+    {
+      "epoch": 0.3736986301369863,
+      "grad_norm": 2.417145252227783,
+      "learning_rate": 4.892363931762794e-06,
+      "loss": 2.0024,
+      "step": 341
+    },
+    {
+      "epoch": 0.3747945205479452,
+      "grad_norm": 2.234590768814087,
+      "learning_rate": 4.8903330625507725e-06,
+      "loss": 2.0837,
+      "step": 342
+    },
+    {
+      "epoch": 0.37589041095890413,
+      "grad_norm": 2.404266595840454,
+      "learning_rate": 4.888302193338749e-06,
+      "loss": 2.018,
+      "step": 343
+    },
+    {
+      "epoch": 0.376986301369863,
+      "grad_norm": 2.364802837371826,
+      "learning_rate": 4.886271324126727e-06,
+      "loss": 1.994,
+      "step": 344
+    },
+    {
+      "epoch": 0.3780821917808219,
+      "grad_norm": 2.2262251377105713,
+      "learning_rate": 4.8842404549147034e-06,
+      "loss": 2.0929,
+      "step": 345
+    },
+    {
+      "epoch": 0.37917808219178084,
+      "grad_norm": 2.4854862689971924,
+      "learning_rate": 4.8822095857026816e-06,
+      "loss": 2.0801,
+      "step": 346
+    },
+    {
+      "epoch": 0.3802739726027397,
+      "grad_norm": 2.3861470222473145,
+      "learning_rate": 4.880178716490658e-06,
+      "loss": 2.0496,
+      "step": 347
+    },
+    {
+      "epoch": 0.38136986301369863,
+      "grad_norm": 2.372326612472534,
+      "learning_rate": 4.878147847278636e-06,
+      "loss": 1.9939,
+      "step": 348
+    },
+    {
+      "epoch": 0.38246575342465755,
+      "grad_norm": 2.3593149185180664,
+      "learning_rate": 4.8761169780666125e-06,
+      "loss": 2.0108,
+      "step": 349
+    },
+    {
+      "epoch": 0.3835616438356164,
+      "grad_norm": 2.1802141666412354,
+      "learning_rate": 4.874086108854591e-06,
+      "loss": 1.983,
+      "step": 350
+    },
+    {
+      "epoch": 0.3835616438356164,
+      "eval_loss": 2.1947567462921143,
+      "eval_runtime": 36.8798,
+      "eval_samples_per_second": 27.115,
+      "eval_steps_per_second": 1.139,
+      "step": 350
+    },
+    {
+      "epoch": 0.38465753424657534,
+      "grad_norm": 2.47067928314209,
+      "learning_rate": 4.872055239642567e-06,
+      "loss": 2.0002,
+      "step": 351
+    },
+    {
+      "epoch": 0.38575342465753426,
+      "grad_norm": 2.359609365463257,
+      "learning_rate": 4.870024370430544e-06,
+      "loss": 1.9799,
+      "step": 352
+    },
+    {
+      "epoch": 0.3868493150684931,
+      "grad_norm": 2.3711702823638916,
+      "learning_rate": 4.867993501218522e-06,
+      "loss": 2.0341,
+      "step": 353
+    },
+    {
+      "epoch": 0.38794520547945205,
+      "grad_norm": 2.246647834777832,
+      "learning_rate": 4.865962632006499e-06,
+      "loss": 2.0687,
+      "step": 354
+    },
+    {
+      "epoch": 0.38904109589041097,
+      "grad_norm": 2.3031938076019287,
+      "learning_rate": 4.863931762794476e-06,
+      "loss": 1.9994,
+      "step": 355
+    },
+    {
+      "epoch": 0.39013698630136984,
+      "grad_norm": 2.492159605026245,
+      "learning_rate": 4.8619008935824535e-06,
+      "loss": 2.0195,
+      "step": 356
+    },
+    {
+      "epoch": 0.39123287671232876,
+      "grad_norm": 2.2878570556640625,
+      "learning_rate": 4.859870024370431e-06,
+      "loss": 2.0589,
+      "step": 357
+    },
+    {
+      "epoch": 0.3923287671232877,
+      "grad_norm": 2.444096088409424,
+      "learning_rate": 4.857839155158408e-06,
+      "loss": 2.0796,
+      "step": 358
+    },
+    {
+      "epoch": 0.3934246575342466,
+      "grad_norm": 2.463857650756836,
+      "learning_rate": 4.855808285946385e-06,
+      "loss": 2.0824,
+      "step": 359
+    },
+    {
+      "epoch": 0.39452054794520547,
+      "grad_norm": 2.0893168449401855,
+      "learning_rate": 4.8537774167343626e-06,
+      "loss": 2.0651,
+      "step": 360
+    },
+    {
+      "epoch": 0.39452054794520547,
+      "eval_loss": 2.188523054122925,
+      "eval_runtime": 36.8679,
+      "eval_samples_per_second": 27.124,
+      "eval_steps_per_second": 1.139,
+      "step": 360
+    },
+    {
+      "epoch": 0.3956164383561644,
+      "grad_norm": 2.602850914001465,
+      "learning_rate": 4.85174654752234e-06,
+      "loss": 2.0507,
+      "step": 361
+    },
+    {
+      "epoch": 0.3967123287671233,
+      "grad_norm": 2.4185140132904053,
+      "learning_rate": 4.849715678310317e-06,
+      "loss": 2.0386,
+      "step": 362
+    },
+    {
+      "epoch": 0.3978082191780822,
+      "grad_norm": 2.1492245197296143,
+      "learning_rate": 4.847684809098294e-06,
+      "loss": 2.0376,
+      "step": 363
+    },
+    {
+      "epoch": 0.3989041095890411,
+      "grad_norm": 2.5710153579711914,
+      "learning_rate": 4.845653939886272e-06,
+      "loss": 2.0383,
+      "step": 364
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 2.4986608028411865,
+      "learning_rate": 4.843623070674249e-06,
+      "loss": 2.0464,
+      "step": 365
+    },
+    {
+      "epoch": 0.4010958904109589,
+      "grad_norm": 2.260388135910034,
+      "learning_rate": 4.841592201462226e-06,
+      "loss": 2.0217,
+      "step": 366
+    },
+    {
+      "epoch": 0.4021917808219178,
+      "grad_norm": 2.6500394344329834,
+      "learning_rate": 4.8395613322502035e-06,
+      "loss": 2.0777,
+      "step": 367
+    },
+    {
+      "epoch": 0.40328767123287673,
+      "grad_norm": 2.159274101257324,
+      "learning_rate": 4.837530463038181e-06,
+      "loss": 1.9747,
+      "step": 368
+    },
+    {
+      "epoch": 0.4043835616438356,
+      "grad_norm": 2.4623043537139893,
+      "learning_rate": 4.835499593826158e-06,
+      "loss": 2.039,
+      "step": 369
+    },
+    {
+      "epoch": 0.4054794520547945,
+      "grad_norm": 2.2265007495880127,
+      "learning_rate": 4.833468724614135e-06,
+      "loss": 2.0618,
+      "step": 370
+    },
+    {
+      "epoch": 0.4054794520547945,
+      "eval_loss": 2.184051275253296,
+      "eval_runtime": 36.9284,
+      "eval_samples_per_second": 27.079,
+      "eval_steps_per_second": 1.137,
+      "step": 370
+    },
+    {
+      "epoch": 0.40657534246575344,
+      "grad_norm": 2.300276517868042,
+      "learning_rate": 4.831437855402113e-06,
+      "loss": 2.0382,
+      "step": 371
+    },
+    {
+      "epoch": 0.4076712328767123,
+      "grad_norm": 2.341156482696533,
+      "learning_rate": 4.82940698619009e-06,
+      "loss": 1.9361,
+      "step": 372
+    },
+    {
+      "epoch": 0.40876712328767123,
+      "grad_norm": 2.4196999073028564,
+      "learning_rate": 4.827376116978067e-06,
+      "loss": 2.0087,
+      "step": 373
+    },
+    {
+      "epoch": 0.40986301369863015,
+      "grad_norm": 2.4650514125823975,
+      "learning_rate": 4.8253452477660444e-06,
+      "loss": 2.0687,
+      "step": 374
+    },
+    {
+      "epoch": 0.410958904109589,
+      "grad_norm": 2.393739700317383,
+      "learning_rate": 4.823314378554022e-06,
+      "loss": 2.0421,
+      "step": 375
+    },
+    {
+      "epoch": 0.41205479452054794,
+      "grad_norm": 2.2436327934265137,
+      "learning_rate": 4.821283509341998e-06,
+      "loss": 2.0358,
+      "step": 376
+    },
+    {
+      "epoch": 0.41315068493150686,
+      "grad_norm": 2.1801650524139404,
+      "learning_rate": 4.819252640129976e-06,
+      "loss": 2.0081,
+      "step": 377
+    },
+    {
+      "epoch": 0.41424657534246573,
+      "grad_norm": 2.300882339477539,
+      "learning_rate": 4.817221770917953e-06,
+      "loss": 2.0321,
+      "step": 378
+    },
+    {
+      "epoch": 0.41534246575342465,
+      "grad_norm": 2.2136058807373047,
+      "learning_rate": 4.815190901705931e-06,
+      "loss": 1.9999,
+      "step": 379
+    },
+    {
+      "epoch": 0.41643835616438357,
+      "grad_norm": 2.168034076690674,
+      "learning_rate": 4.813160032493907e-06,
+      "loss": 2.0336,
+      "step": 380
+    },
+    {
+      "epoch": 0.41643835616438357,
+      "eval_loss": 2.1770179271698,
+      "eval_runtime": 37.0015,
+      "eval_samples_per_second": 27.026,
+      "eval_steps_per_second": 1.135,
+      "step": 380
+    },
+    {
+      "epoch": 0.41753424657534244,
+      "grad_norm": 2.1604163646698,
+      "learning_rate": 4.811129163281885e-06,
+      "loss": 2.0212,
+      "step": 381
+    },
+    {
+      "epoch": 0.41863013698630136,
+      "grad_norm": 2.23923659324646,
+      "learning_rate": 4.809098294069862e-06,
+      "loss": 2.0495,
+      "step": 382
+    },
+    {
+      "epoch": 0.4197260273972603,
+      "grad_norm": 2.149949312210083,
+      "learning_rate": 4.80706742485784e-06,
+      "loss": 2.041,
+      "step": 383
+    },
+    {
+      "epoch": 0.4208219178082192,
+      "grad_norm": 2.49120831489563,
+      "learning_rate": 4.805036555645816e-06,
+      "loss": 2.0931,
+      "step": 384
+    },
+    {
+      "epoch": 0.42191780821917807,
+      "grad_norm": 2.2123563289642334,
+      "learning_rate": 4.8030056864337945e-06,
+      "loss": 2.0087,
+      "step": 385
+    },
+    {
+      "epoch": 0.423013698630137,
+      "grad_norm": 2.4235594272613525,
+      "learning_rate": 4.800974817221771e-06,
+      "loss": 2.0656,
+      "step": 386
+    },
+    {
+      "epoch": 0.4241095890410959,
+      "grad_norm": 2.158252239227295,
+      "learning_rate": 4.798943948009749e-06,
+      "loss": 2.0181,
+      "step": 387
+    },
+    {
+      "epoch": 0.4252054794520548,
+      "grad_norm": 2.2412970066070557,
+      "learning_rate": 4.7969130787977254e-06,
+      "loss": 2.0323,
+      "step": 388
+    },
+    {
+      "epoch": 0.4263013698630137,
+      "grad_norm": 2.2520956993103027,
+      "learning_rate": 4.794882209585703e-06,
+      "loss": 2.026,
+      "step": 389
+    },
+    {
+      "epoch": 0.4273972602739726,
+      "grad_norm": 2.168954849243164,
+      "learning_rate": 4.79285134037368e-06,
+      "loss": 2.0459,
+      "step": 390
+    },
+    {
+      "epoch": 0.4273972602739726,
+      "eval_loss": 2.1705472469329834,
+      "eval_runtime": 37.0913,
+      "eval_samples_per_second": 26.96,
+      "eval_steps_per_second": 1.132,
+      "step": 390
+    },
+    {
+      "epoch": 0.4284931506849315,
+      "grad_norm": 2.1397407054901123,
+      "learning_rate": 4.790820471161657e-06,
+      "loss": 1.9995,
+      "step": 391
+    },
+    {
+      "epoch": 0.4295890410958904,
+      "grad_norm": 2.1747751235961914,
+      "learning_rate": 4.7887896019496345e-06,
+      "loss": 2.0088,
+      "step": 392
+    },
+    {
+      "epoch": 0.43068493150684933,
+      "grad_norm": 2.2860991954803467,
+      "learning_rate": 4.786758732737612e-06,
+      "loss": 2.0134,
+      "step": 393
+    },
+    {
+      "epoch": 0.4317808219178082,
+      "grad_norm": 2.2133519649505615,
+      "learning_rate": 4.784727863525589e-06,
+      "loss": 2.0586,
+      "step": 394
+    },
+    {
+      "epoch": 0.4328767123287671,
+      "grad_norm": 2.2147040367126465,
+      "learning_rate": 4.782696994313566e-06,
+      "loss": 1.9748,
+      "step": 395
+    },
+    {
+      "epoch": 0.43397260273972604,
+      "grad_norm": 2.343418836593628,
+      "learning_rate": 4.780666125101544e-06,
+      "loss": 2.0223,
+      "step": 396
+    },
+    {
+      "epoch": 0.4350684931506849,
+      "grad_norm": 2.198935031890869,
+      "learning_rate": 4.778635255889521e-06,
+      "loss": 1.9627,
+      "step": 397
+    },
+    {
+      "epoch": 0.43616438356164383,
+      "grad_norm": 2.1314704418182373,
+      "learning_rate": 4.776604386677498e-06,
+      "loss": 1.9938,
+      "step": 398
+    },
+    {
+      "epoch": 0.43726027397260275,
+      "grad_norm": 2.3005123138427734,
+      "learning_rate": 4.7745735174654755e-06,
+      "loss": 2.0029,
+      "step": 399
+    },
+    {
+      "epoch": 0.4383561643835616,
+      "grad_norm": 2.2826664447784424,
+      "learning_rate": 4.772542648253453e-06,
+      "loss": 2.0585,
+      "step": 400
+    },
+    {
+      "epoch": 0.4383561643835616,
+      "eval_loss": 2.1691415309906006,
+      "eval_runtime": 37.0852,
+      "eval_samples_per_second": 26.965,
+      "eval_steps_per_second": 1.133,
+      "step": 400
+    },
+    {
+      "epoch": 0.43945205479452054,
+      "grad_norm": 2.0547878742218018,
+      "learning_rate": 4.77051177904143e-06,
+      "loss": 2.0582,
+      "step": 401
+    },
+    {
+      "epoch": 0.44054794520547946,
+      "grad_norm": 2.200042486190796,
+      "learning_rate": 4.768480909829407e-06,
+      "loss": 1.9936,
+      "step": 402
+    },
+    {
+      "epoch": 0.44164383561643833,
+      "grad_norm": 2.354637861251831,
+      "learning_rate": 4.766450040617385e-06,
+      "loss": 2.0327,
+      "step": 403
+    },
+    {
+      "epoch": 0.44273972602739725,
+      "grad_norm": 2.2332348823547363,
+      "learning_rate": 4.764419171405362e-06,
+      "loss": 2.0052,
+      "step": 404
+    },
+    {
+      "epoch": 0.4438356164383562,
+      "grad_norm": 2.288060426712036,
+      "learning_rate": 4.762388302193339e-06,
+      "loss": 2.0472,
+      "step": 405
+    },
+    {
+      "epoch": 0.4449315068493151,
+      "grad_norm": 2.1368887424468994,
+      "learning_rate": 4.760357432981316e-06,
+      "loss": 2.0305,
+      "step": 406
+    },
+    {
+      "epoch": 0.44602739726027396,
+      "grad_norm": 2.362630844116211,
+      "learning_rate": 4.758326563769294e-06,
+      "loss": 2.002,
+      "step": 407
+    },
+    {
+      "epoch": 0.4471232876712329,
+      "grad_norm": 2.283419370651245,
+      "learning_rate": 4.756295694557271e-06,
+      "loss": 2.0271,
+      "step": 408
+    },
+    {
+      "epoch": 0.4482191780821918,
+      "grad_norm": 2.0400819778442383,
+      "learning_rate": 4.754264825345248e-06,
+      "loss": 2.0256,
+      "step": 409
+    },
+    {
+      "epoch": 0.44931506849315067,
+      "grad_norm": 2.588416814804077,
+      "learning_rate": 4.7522339561332255e-06,
+      "loss": 1.9756,
+      "step": 410
+    },
+    {
+      "epoch": 0.44931506849315067,
+      "eval_loss": 2.17041015625,
+      "eval_runtime": 36.958,
+      "eval_samples_per_second": 27.058,
+      "eval_steps_per_second": 1.136,
+      "step": 410
+    },
+    {
+      "epoch": 0.4504109589041096,
+      "grad_norm": 2.2397494316101074,
+      "learning_rate": 4.750203086921203e-06,
+      "loss": 1.9876,
+      "step": 411
+    },
+    {
+      "epoch": 0.4515068493150685,
+      "grad_norm": 2.2085509300231934,
+      "learning_rate": 4.74817221770918e-06,
+      "loss": 1.9464,
+      "step": 412
+    },
+    {
+      "epoch": 0.4526027397260274,
+      "grad_norm": 2.140974760055542,
+      "learning_rate": 4.7461413484971565e-06,
+      "loss": 2.0179,
+      "step": 413
+    },
+    {
+      "epoch": 0.4536986301369863,
+      "grad_norm": 2.124366044998169,
+      "learning_rate": 4.744110479285135e-06,
+      "loss": 2.0732,
+      "step": 414
+    },
+    {
+      "epoch": 0.4547945205479452,
+      "grad_norm": 2.2463996410369873,
+      "learning_rate": 4.742079610073111e-06,
+      "loss": 1.9616,
+      "step": 415
+    },
+    {
+      "epoch": 0.4558904109589041,
+      "grad_norm": 2.1249961853027344,
+      "learning_rate": 4.740048740861089e-06,
+      "loss": 1.9889,
+      "step": 416
+    },
+    {
+      "epoch": 0.456986301369863,
+      "grad_norm": 2.25842022895813,
+      "learning_rate": 4.738017871649066e-06,
+      "loss": 2.0033,
+      "step": 417
+    },
+    {
+      "epoch": 0.45808219178082193,
+      "grad_norm": 2.271935224533081,
+      "learning_rate": 4.735987002437044e-06,
+      "loss": 2.0148,
+      "step": 418
+    },
+    {
+      "epoch": 0.4591780821917808,
+      "grad_norm": 2.4797163009643555,
+      "learning_rate": 4.73395613322502e-06,
+      "loss": 2.0736,
+      "step": 419
+    },
+    {
+      "epoch": 0.4602739726027397,
+      "grad_norm": 2.1901395320892334,
+      "learning_rate": 4.731925264012998e-06,
+      "loss": 2.0763,
+      "step": 420
+    },
+    {
+      "epoch": 0.4602739726027397,
+      "eval_loss": 2.161660671234131,
+      "eval_runtime": 36.9099,
+      "eval_samples_per_second": 27.093,
+      "eval_steps_per_second": 1.138,
+      "step": 420
+    },
+    {
+      "epoch": 0.46136986301369864,
+      "grad_norm": 2.083536386489868,
+      "learning_rate": 4.729894394800975e-06,
+      "loss": 2.0299,
+      "step": 421
+    },
+    {
+      "epoch": 0.4624657534246575,
+      "grad_norm": 2.1118173599243164,
+      "learning_rate": 4.727863525588953e-06,
+      "loss": 1.9742,
+      "step": 422
+    },
+    {
+      "epoch": 0.46356164383561643,
+      "grad_norm": 2.195802927017212,
+      "learning_rate": 4.725832656376929e-06,
+      "loss": 1.9318,
+      "step": 423
+    },
+    {
+      "epoch": 0.46465753424657535,
+      "grad_norm": 2.318098545074463,
+      "learning_rate": 4.723801787164907e-06,
+      "loss": 1.9875,
+      "step": 424
+    },
+    {
+      "epoch": 0.4657534246575342,
+      "grad_norm": 2.1698496341705322,
+      "learning_rate": 4.721770917952885e-06,
+      "loss": 2.031,
+      "step": 425
+    },
+    {
+      "epoch": 0.46684931506849314,
+      "grad_norm": 2.283935785293579,
+      "learning_rate": 4.719740048740861e-06,
+      "loss": 1.9766,
+      "step": 426
+    },
+    {
+      "epoch": 0.46794520547945206,
+      "grad_norm": 2.4180757999420166,
+      "learning_rate": 4.717709179528839e-06,
+      "loss": 1.9976,
+      "step": 427
+    },
+    {
+      "epoch": 0.469041095890411,
+      "grad_norm": 2.2689030170440674,
+      "learning_rate": 4.715678310316816e-06,
+      "loss": 2.03,
+      "step": 428
+    },
+    {
+      "epoch": 0.47013698630136985,
+      "grad_norm": 2.423708200454712,
+      "learning_rate": 4.713647441104794e-06,
+      "loss": 2.0202,
+      "step": 429
+    },
+    {
+      "epoch": 0.4712328767123288,
+      "grad_norm": 2.184705972671509,
+      "learning_rate": 4.71161657189277e-06,
+      "loss": 1.9655,
+      "step": 430
+    },
+    {
+      "epoch": 0.4712328767123288,
+      "eval_loss": 2.1548638343811035,
+      "eval_runtime": 36.9426,
+      "eval_samples_per_second": 27.069,
+      "eval_steps_per_second": 1.137,
+      "step": 430
+    },
+    {
+      "epoch": 0.4723287671232877,
+      "grad_norm": 2.295851945877075,
+      "learning_rate": 4.709585702680748e-06,
+      "loss": 1.9561,
+      "step": 431
+    },
+    {
+      "epoch": 0.47342465753424656,
+      "grad_norm": 2.1762046813964844,
+      "learning_rate": 4.707554833468725e-06,
+      "loss": 2.0341,
+      "step": 432
+    },
+    {
+      "epoch": 0.4745205479452055,
+      "grad_norm": 2.4684343338012695,
+      "learning_rate": 4.705523964256703e-06,
+      "loss": 2.0898,
+      "step": 433
+    },
+    {
+      "epoch": 0.4756164383561644,
+      "grad_norm": 2.293318271636963,
+      "learning_rate": 4.703493095044679e-06,
+      "loss": 1.9794,
+      "step": 434
+    },
+    {
+      "epoch": 0.4767123287671233,
+      "grad_norm": 2.1884210109710693,
+      "learning_rate": 4.701462225832657e-06,
+      "loss": 1.9979,
+      "step": 435
+    },
+    {
+      "epoch": 0.4778082191780822,
+      "grad_norm": 2.412504196166992,
+      "learning_rate": 4.699431356620634e-06,
+      "loss": 1.9237,
+      "step": 436
+    },
+    {
+      "epoch": 0.4789041095890411,
+      "grad_norm": 2.192824602127075,
+      "learning_rate": 4.697400487408612e-06,
+      "loss": 2.0756,
+      "step": 437
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 2.368100643157959,
+      "learning_rate": 4.695369618196588e-06,
+      "loss": 2.028,
+      "step": 438
+    },
+    {
+      "epoch": 0.4810958904109589,
+      "grad_norm": 2.218454122543335,
+      "learning_rate": 4.693338748984566e-06,
+      "loss": 2.0049,
+      "step": 439
+    },
+    {
+      "epoch": 0.4821917808219178,
+      "grad_norm": 2.356168031692505,
+      "learning_rate": 4.691307879772543e-06,
+      "loss": 1.9958,
+      "step": 440
+    },
+    {
+      "epoch": 0.4821917808219178,
+      "eval_loss": 2.1540727615356445,
+      "eval_runtime": 36.9557,
+      "eval_samples_per_second": 27.059,
+      "eval_steps_per_second": 1.136,
+      "step": 440
+    },
+    {
+      "epoch": 0.4832876712328767,
+      "grad_norm": 2.09956693649292,
+      "learning_rate": 4.68927701056052e-06,
+      "loss": 1.9388,
+      "step": 441
+    },
+    {
+      "epoch": 0.4843835616438356,
+      "grad_norm": 2.5746614933013916,
+      "learning_rate": 4.6872461413484975e-06,
+      "loss": 2.0425,
+      "step": 442
+    },
+    {
+      "epoch": 0.48547945205479454,
+      "grad_norm": 2.2064783573150635,
+      "learning_rate": 4.685215272136475e-06,
+      "loss": 2.0082,
+      "step": 443
+    },
+    {
+      "epoch": 0.4865753424657534,
+      "grad_norm": 2.159029006958008,
+      "learning_rate": 4.683184402924452e-06,
+      "loss": 2.0055,
+      "step": 444
+    },
+    {
+      "epoch": 0.4876712328767123,
+      "grad_norm": 2.5222008228302,
+      "learning_rate": 4.681153533712429e-06,
+      "loss": 2.0007,
+      "step": 445
+    },
+    {
+      "epoch": 0.48876712328767125,
+      "grad_norm": 2.163487672805786,
+      "learning_rate": 4.679122664500407e-06,
+      "loss": 2.0719,
+      "step": 446
+    },
+    {
+      "epoch": 0.4898630136986301,
+      "grad_norm": 2.3383803367614746,
+      "learning_rate": 4.677091795288384e-06,
+      "loss": 1.9623,
+      "step": 447
+    },
+    {
+      "epoch": 0.49095890410958903,
+      "grad_norm": 2.1056675910949707,
+      "learning_rate": 4.675060926076361e-06,
+      "loss": 1.9443,
+      "step": 448
+    },
+    {
+      "epoch": 0.49205479452054796,
+      "grad_norm": 2.145270824432373,
+      "learning_rate": 4.673030056864338e-06,
+      "loss": 2.0469,
+      "step": 449
+    },
+    {
+      "epoch": 0.4931506849315068,
+      "grad_norm": 2.578706741333008,
+      "learning_rate": 4.670999187652316e-06,
+      "loss": 1.9569,
+      "step": 450
+    },
+    {
+      "epoch": 0.4931506849315068,
+      "eval_loss": 2.1527743339538574,
+      "eval_runtime": 36.8961,
+      "eval_samples_per_second": 27.103,
+      "eval_steps_per_second": 1.138,
+      "step": 450
+    },
+    {
+      "epoch": 0.49424657534246574,
+      "grad_norm": 2.1328542232513428,
+      "learning_rate": 4.668968318440293e-06,
+      "loss": 1.9559,
+      "step": 451
+    },
+    {
+      "epoch": 0.49534246575342467,
+      "grad_norm": 2.148820638656616,
+      "learning_rate": 4.66693744922827e-06,
+      "loss": 1.9545,
+      "step": 452
+    },
+    {
+      "epoch": 0.4964383561643836,
+      "grad_norm": 2.250340461730957,
+      "learning_rate": 4.6649065800162475e-06,
+      "loss": 2.0358,
+      "step": 453
+    },
+    {
+      "epoch": 0.49753424657534245,
+      "grad_norm": 2.452397584915161,
+      "learning_rate": 4.662875710804225e-06,
+      "loss": 1.9294,
+      "step": 454
+    },
+    {
+      "epoch": 0.4986301369863014,
+      "grad_norm": 2.140615701675415,
+      "learning_rate": 4.660844841592202e-06,
+      "loss": 2.0084,
+      "step": 455
+    },
+    {
+      "epoch": 0.4997260273972603,
+      "grad_norm": 2.176806926727295,
+      "learning_rate": 4.658813972380179e-06,
+      "loss": 1.9174,
+      "step": 456
+    },
+    {
+      "epoch": 0.5008219178082192,
+      "grad_norm": 2.349031925201416,
+      "learning_rate": 4.656783103168157e-06,
+      "loss": 2.0377,
+      "step": 457
+    },
+    {
+      "epoch": 0.5019178082191781,
+      "grad_norm": 2.436889171600342,
+      "learning_rate": 4.654752233956134e-06,
+      "loss": 1.9133,
+      "step": 458
+    },
+    {
+      "epoch": 0.503013698630137,
+      "grad_norm": 2.116610288619995,
+      "learning_rate": 4.652721364744111e-06,
+      "loss": 1.9993,
+      "step": 459
+    },
+    {
+      "epoch": 0.5041095890410959,
+      "grad_norm": 2.186915636062622,
+      "learning_rate": 4.6506904955320885e-06,
+      "loss": 1.9744,
+      "step": 460
+    },
+    {
+      "epoch": 0.5041095890410959,
+      "eval_loss": 2.1580278873443604,
+      "eval_runtime": 36.9097,
+      "eval_samples_per_second": 27.093,
+      "eval_steps_per_second": 1.138,
+      "step": 460
+    },
+    {
+      "epoch": 0.5052054794520547,
+      "grad_norm": 2.2009212970733643,
+      "learning_rate": 4.648659626320066e-06,
+      "loss": 2.0173,
+      "step": 461
+    },
+    {
+      "epoch": 0.5063013698630137,
+      "grad_norm": 2.117286443710327,
+      "learning_rate": 4.646628757108043e-06,
+      "loss": 2.0238,
+      "step": 462
+    },
+    {
+      "epoch": 0.5073972602739726,
+      "grad_norm": 2.238790273666382,
+      "learning_rate": 4.6445978878960194e-06,
+      "loss": 1.9885,
+      "step": 463
+    },
+    {
+      "epoch": 0.5084931506849315,
+      "grad_norm": 2.3493096828460693,
+      "learning_rate": 4.6425670186839976e-06,
+      "loss": 1.9593,
+      "step": 464
+    },
+    {
+      "epoch": 0.5095890410958904,
+      "grad_norm": 2.3276610374450684,
+      "learning_rate": 4.640536149471974e-06,
+      "loss": 1.9993,
+      "step": 465
+    },
+    {
+      "epoch": 0.5106849315068493,
+      "grad_norm": 2.3671715259552,
+      "learning_rate": 4.638505280259952e-06,
+      "loss": 2.0256,
+      "step": 466
+    },
+    {
+      "epoch": 0.5117808219178083,
+      "grad_norm": 2.251803159713745,
+      "learning_rate": 4.6364744110479285e-06,
+      "loss": 2.0436,
+      "step": 467
+    },
+    {
+      "epoch": 0.5128767123287671,
+      "grad_norm": 2.3023312091827393,
+      "learning_rate": 4.634443541835907e-06,
+      "loss": 1.9968,
+      "step": 468
+    },
+    {
+      "epoch": 0.513972602739726,
+      "grad_norm": 2.3660333156585693,
+      "learning_rate": 4.632412672623883e-06,
+      "loss": 2.0492,
+      "step": 469
+    },
+    {
+      "epoch": 0.5150684931506849,
+      "grad_norm": 2.262179136276245,
+      "learning_rate": 4.630381803411861e-06,
+      "loss": 1.9698,
+      "step": 470
+    },
+    {
+      "epoch": 0.5150684931506849,
+      "eval_loss": 2.1516528129577637,
+      "eval_runtime": 36.7956,
+      "eval_samples_per_second": 27.177,
+      "eval_steps_per_second": 1.141,
+      "step": 470
+    },
+    {
+      "epoch": 0.5161643835616438,
+      "grad_norm": 2.450770616531372,
+      "learning_rate": 4.628350934199838e-06,
+      "loss": 1.8799,
+      "step": 471
+    },
+    {
+      "epoch": 0.5172602739726028,
+      "grad_norm": 2.187458038330078,
+      "learning_rate": 4.626320064987816e-06,
+      "loss": 1.9829,
+      "step": 472
+    },
+    {
+      "epoch": 0.5183561643835617,
+      "grad_norm": 2.1726722717285156,
+      "learning_rate": 4.624289195775792e-06,
+      "loss": 2.012,
+      "step": 473
+    },
+    {
+      "epoch": 0.5194520547945205,
+      "grad_norm": 2.4177136421203613,
+      "learning_rate": 4.62225832656377e-06,
+      "loss": 1.9796,
+      "step": 474
+    },
+    {
+      "epoch": 0.5205479452054794,
+      "grad_norm": 2.2353951930999756,
+      "learning_rate": 4.620227457351747e-06,
+      "loss": 1.99,
+      "step": 475
+    },
+    {
+      "epoch": 0.5216438356164383,
+      "grad_norm": 2.369591236114502,
+      "learning_rate": 4.618196588139724e-06,
+      "loss": 1.9847,
+      "step": 476
+    },
+    {
+      "epoch": 0.5227397260273973,
+      "grad_norm": 2.546703577041626,
+      "learning_rate": 4.616165718927701e-06,
+      "loss": 2.0047,
+      "step": 477
+    },
+    {
+      "epoch": 0.5238356164383562,
+      "grad_norm": 2.3509185314178467,
+      "learning_rate": 4.6141348497156786e-06,
+      "loss": 1.9589,
+      "step": 478
+    },
+    {
+      "epoch": 0.5249315068493151,
+      "grad_norm": 2.3063488006591797,
+      "learning_rate": 4.612103980503656e-06,
+      "loss": 2.045,
+      "step": 479
+    },
+    {
+      "epoch": 0.5260273972602739,
+      "grad_norm": 2.285572052001953,
+      "learning_rate": 4.610073111291633e-06,
+      "loss": 2.0218,
+      "step": 480
+    },
+    {
+      "epoch": 0.5260273972602739,
+      "eval_loss": 2.1472301483154297,
+      "eval_runtime": 36.7689,
+      "eval_samples_per_second": 27.197,
+      "eval_steps_per_second": 1.142,
+      "step": 480
+    },
+    {
+      "epoch": 0.5271232876712328,
+      "grad_norm": 2.22987961769104,
+      "learning_rate": 4.60804224207961e-06,
+      "loss": 1.9955,
+      "step": 481
+    },
+    {
+      "epoch": 0.5282191780821918,
+      "grad_norm": 2.200918197631836,
+      "learning_rate": 4.606011372867588e-06,
+      "loss": 1.9768,
+      "step": 482
+    },
+    {
+      "epoch": 0.5293150684931507,
+      "grad_norm": 2.2441930770874023,
+      "learning_rate": 4.603980503655565e-06,
+      "loss": 2.0088,
+      "step": 483
+    },
+    {
+      "epoch": 0.5304109589041096,
+      "grad_norm": 2.380383014678955,
+      "learning_rate": 4.601949634443542e-06,
+      "loss": 1.9853,
+      "step": 484
+    },
+    {
+      "epoch": 0.5315068493150685,
+      "grad_norm": 2.317995071411133,
+      "learning_rate": 4.5999187652315195e-06,
+      "loss": 1.9183,
+      "step": 485
+    },
+    {
+      "epoch": 0.5326027397260275,
+      "grad_norm": 2.2462668418884277,
+      "learning_rate": 4.597887896019497e-06,
+      "loss": 1.9201,
+      "step": 486
+    },
+    {
+      "epoch": 0.5336986301369863,
+      "grad_norm": 2.1482582092285156,
+      "learning_rate": 4.595857026807474e-06,
+      "loss": 1.9722,
+      "step": 487
+    },
+    {
+      "epoch": 0.5347945205479452,
+      "grad_norm": 2.079831123352051,
+      "learning_rate": 4.593826157595451e-06,
+      "loss": 1.977,
+      "step": 488
+    },
+    {
+      "epoch": 0.5358904109589041,
+      "grad_norm": 2.2169578075408936,
+      "learning_rate": 4.591795288383429e-06,
+      "loss": 1.9615,
+      "step": 489
+    },
+    {
+      "epoch": 0.536986301369863,
+      "grad_norm": 2.087846279144287,
+      "learning_rate": 4.589764419171406e-06,
+      "loss": 2.0402,
+      "step": 490
+    },
+    {
+      "epoch": 0.536986301369863,
+      "eval_loss": 2.1416702270507812,
+      "eval_runtime": 37.1191,
+      "eval_samples_per_second": 26.94,
+      "eval_steps_per_second": 1.131,
+      "step": 490
+    },
+    {
+      "epoch": 0.538082191780822,
+      "grad_norm": 2.2161977291107178,
+      "learning_rate": 4.587733549959383e-06,
+      "loss": 1.9959,
+      "step": 491
+    },
+    {
+      "epoch": 0.5391780821917809,
+      "grad_norm": 2.19530987739563,
+      "learning_rate": 4.5857026807473604e-06,
+      "loss": 1.961,
+      "step": 492
+    },
+    {
+      "epoch": 0.5402739726027397,
+      "grad_norm": 2.1150615215301514,
+      "learning_rate": 4.583671811535338e-06,
+      "loss": 1.946,
+      "step": 493
+    },
+    {
+      "epoch": 0.5413698630136986,
+      "grad_norm": 2.186352491378784,
+      "learning_rate": 4.581640942323315e-06,
+      "loss": 1.9137,
+      "step": 494
+    },
+    {
+      "epoch": 0.5424657534246575,
+      "grad_norm": 2.233370780944824,
+      "learning_rate": 4.579610073111292e-06,
+      "loss": 2.021,
+      "step": 495
+    },
+    {
+      "epoch": 0.5435616438356164,
+      "grad_norm": 2.1326382160186768,
+      "learning_rate": 4.5775792038992695e-06,
+      "loss": 2.0242,
+      "step": 496
+    },
+    {
+      "epoch": 0.5446575342465754,
+      "grad_norm": 2.536837577819824,
+      "learning_rate": 4.575548334687247e-06,
+      "loss": 2.0185,
+      "step": 497
+    },
+    {
+      "epoch": 0.5457534246575343,
+      "grad_norm": 2.193598747253418,
+      "learning_rate": 4.573517465475224e-06,
+      "loss": 2.0533,
+      "step": 498
+    },
+    {
+      "epoch": 0.5468493150684931,
+      "grad_norm": 2.1346311569213867,
+      "learning_rate": 4.571486596263201e-06,
+      "loss": 1.9564,
+      "step": 499
+    },
+    {
+      "epoch": 0.547945205479452,
+      "grad_norm": 2.4294722080230713,
+      "learning_rate": 4.569455727051178e-06,
+      "loss": 2.0003,
+      "step": 500
+    },
+    {
+      "epoch": 0.547945205479452,
+      "eval_loss": 2.1430764198303223,
+      "eval_runtime": 37.0591,
+      "eval_samples_per_second": 26.984,
+      "eval_steps_per_second": 1.133,
+      "step": 500
+    },
+    {
+      "epoch": 0.549041095890411,
+      "grad_norm": 2.2394182682037354,
+      "learning_rate": 4.567424857839156e-06,
+      "loss": 2.0055,
+      "step": 501
+    },
+    {
+      "epoch": 0.5501369863013699,
+      "grad_norm": 2.2543067932128906,
+      "learning_rate": 4.565393988627132e-06,
+      "loss": 1.9837,
+      "step": 502
+    },
+    {
+      "epoch": 0.5512328767123288,
+      "grad_norm": 2.2805464267730713,
+      "learning_rate": 4.5633631194151105e-06,
+      "loss": 2.0495,
+      "step": 503
+    },
+    {
+      "epoch": 0.5523287671232877,
+      "grad_norm": 2.2838146686553955,
+      "learning_rate": 4.561332250203087e-06,
+      "loss": 1.9866,
+      "step": 504
+    },
+    {
+      "epoch": 0.5534246575342465,
+      "grad_norm": 2.286147356033325,
+      "learning_rate": 4.559301380991065e-06,
+      "loss": 1.9805,
+      "step": 505
+    },
+    {
+      "epoch": 0.5545205479452054,
+      "grad_norm": 2.064378023147583,
+      "learning_rate": 4.5572705117790414e-06,
+      "loss": 1.9259,
+      "step": 506
+    },
+    {
+      "epoch": 0.5556164383561644,
+      "grad_norm": 2.4020493030548096,
+      "learning_rate": 4.5552396425670196e-06,
+      "loss": 1.9731,
+      "step": 507
+    },
+    {
+      "epoch": 0.5567123287671233,
+      "grad_norm": 2.11594557762146,
+      "learning_rate": 4.553208773354996e-06,
+      "loss": 2.0126,
+      "step": 508
+    },
+    {
+      "epoch": 0.5578082191780822,
+      "grad_norm": 2.4520673751831055,
+      "learning_rate": 4.551177904142974e-06,
+      "loss": 1.9396,
+      "step": 509
+    },
+    {
+      "epoch": 0.5589041095890411,
+      "grad_norm": 2.302771806716919,
+      "learning_rate": 4.5491470349309505e-06,
+      "loss": 2.0361,
+      "step": 510
+    },
+    {
+      "epoch": 0.5589041095890411,
+      "eval_loss": 2.1408755779266357,
+      "eval_runtime": 36.9881,
+      "eval_samples_per_second": 27.036,
+      "eval_steps_per_second": 1.136,
+      "step": 510
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 2.250927686691284,
+      "learning_rate": 4.547116165718929e-06,
+      "loss": 1.9616,
+      "step": 511
+    },
+    {
+      "epoch": 0.5610958904109589,
+      "grad_norm": 2.4257938861846924,
+      "learning_rate": 4.545085296506905e-06,
+      "loss": 1.9836,
+      "step": 512
+    },
+    {
+      "epoch": 0.5621917808219178,
+      "grad_norm": 2.08693265914917,
+      "learning_rate": 4.543054427294882e-06,
+      "loss": 1.9541,
+      "step": 513
+    },
+    {
+      "epoch": 0.5632876712328767,
+      "grad_norm": 2.133875846862793,
+      "learning_rate": 4.54102355808286e-06,
+      "loss": 1.9377,
+      "step": 514
+    },
+    {
+      "epoch": 0.5643835616438356,
+      "grad_norm": 2.1217594146728516,
+      "learning_rate": 4.538992688870837e-06,
+      "loss": 1.9369,
+      "step": 515
+    },
+    {
+      "epoch": 0.5654794520547946,
+      "grad_norm": 2.117994546890259,
+      "learning_rate": 4.536961819658814e-06,
+      "loss": 1.9666,
+      "step": 516
+    },
+    {
+      "epoch": 0.5665753424657535,
+      "grad_norm": 2.1514523029327393,
+      "learning_rate": 4.5349309504467915e-06,
+      "loss": 1.9704,
+      "step": 517
+    },
+    {
+      "epoch": 0.5676712328767123,
+      "grad_norm": 2.2309377193450928,
+      "learning_rate": 4.532900081234769e-06,
+      "loss": 1.9858,
+      "step": 518
+    },
+    {
+      "epoch": 0.5687671232876712,
+      "grad_norm": 2.100398540496826,
+      "learning_rate": 4.530869212022746e-06,
+      "loss": 1.9573,
+      "step": 519
+    },
+    {
+      "epoch": 0.5698630136986301,
+      "grad_norm": 2.122840166091919,
+      "learning_rate": 4.528838342810723e-06,
+      "loss": 1.9813,
+      "step": 520
+    },
+    {
+      "epoch": 0.5698630136986301,
+      "eval_loss": 2.135481357574463,
+      "eval_runtime": 36.893,
+      "eval_samples_per_second": 27.105,
+      "eval_steps_per_second": 1.138,
+      "step": 520
+    },
+    {
+      "epoch": 0.570958904109589,
+      "grad_norm": 2.1103999614715576,
+      "learning_rate": 4.526807473598701e-06,
+      "loss": 1.9691,
+      "step": 521
+    },
+    {
+      "epoch": 0.572054794520548,
+      "grad_norm": 2.059232473373413,
+      "learning_rate": 4.524776604386678e-06,
+      "loss": 1.9687,
+      "step": 522
+    },
+    {
+      "epoch": 0.5731506849315069,
+      "grad_norm": 2.207000732421875,
+      "learning_rate": 4.522745735174655e-06,
+      "loss": 1.949,
+      "step": 523
+    },
+    {
+      "epoch": 0.5742465753424657,
+      "grad_norm": 2.264000177383423,
+      "learning_rate": 4.520714865962632e-06,
+      "loss": 2.0066,
+      "step": 524
+    },
+    {
+      "epoch": 0.5753424657534246,
+      "grad_norm": 2.121701717376709,
+      "learning_rate": 4.51868399675061e-06,
+      "loss": 1.9879,
+      "step": 525
+    },
+    {
+      "epoch": 0.5764383561643835,
+      "grad_norm": 2.0745644569396973,
+      "learning_rate": 4.516653127538587e-06,
+      "loss": 1.9924,
+      "step": 526
+    },
+    {
+      "epoch": 0.5775342465753425,
+      "grad_norm": 2.2257349491119385,
+      "learning_rate": 4.514622258326564e-06,
+      "loss": 1.9525,
+      "step": 527
+    },
+    {
+      "epoch": 0.5786301369863014,
+      "grad_norm": 2.1678507328033447,
+      "learning_rate": 4.5125913891145415e-06,
+      "loss": 1.9378,
+      "step": 528
+    },
+    {
+      "epoch": 0.5797260273972603,
+      "grad_norm": 2.2800841331481934,
+      "learning_rate": 4.510560519902519e-06,
+      "loss": 1.9685,
+      "step": 529
+    },
+    {
+      "epoch": 0.5808219178082191,
+      "grad_norm": 2.1471869945526123,
+      "learning_rate": 4.508529650690496e-06,
+      "loss": 2.0027,
+      "step": 530
+    },
+    {
+      "epoch": 0.5808219178082191,
+      "eval_loss": 2.1354892253875732,
+      "eval_runtime": 36.8434,
+      "eval_samples_per_second": 27.142,
+      "eval_steps_per_second": 1.14,
+      "step": 530
+    },
+    {
+      "epoch": 0.581917808219178,
+      "grad_norm": 2.2238621711730957,
+      "learning_rate": 4.506498781478473e-06,
+      "loss": 1.9258,
+      "step": 531
+    },
+    {
+      "epoch": 0.583013698630137,
+      "grad_norm": 2.2224624156951904,
+      "learning_rate": 4.504467912266451e-06,
+      "loss": 1.9168,
+      "step": 532
+    },
+    {
+      "epoch": 0.5841095890410959,
+      "grad_norm": 1.9850294589996338,
+      "learning_rate": 4.502437043054428e-06,
+      "loss": 1.9656,
+      "step": 533
+    },
+    {
+      "epoch": 0.5852054794520548,
+      "grad_norm": 2.1303517818450928,
+      "learning_rate": 4.500406173842405e-06,
+      "loss": 2.0349,
+      "step": 534
+    },
+    {
+      "epoch": 0.5863013698630137,
+      "grad_norm": 2.2114832401275635,
+      "learning_rate": 4.4983753046303824e-06,
+      "loss": 1.9878,
+      "step": 535
+    },
+    {
+      "epoch": 0.5873972602739727,
+      "grad_norm": 1.9375886917114258,
+      "learning_rate": 4.49634443541836e-06,
+      "loss": 1.9724,
+      "step": 536
+    },
+    {
+      "epoch": 0.5884931506849315,
+      "grad_norm": 2.185518741607666,
+      "learning_rate": 4.494313566206336e-06,
+      "loss": 1.9584,
+      "step": 537
+    },
+    {
+      "epoch": 0.5895890410958904,
+      "grad_norm": 2.125638484954834,
+      "learning_rate": 4.492282696994314e-06,
+      "loss": 2.0203,
+      "step": 538
+    },
+    {
+      "epoch": 0.5906849315068493,
+      "grad_norm": 1.9456262588500977,
+      "learning_rate": 4.490251827782291e-06,
+      "loss": 1.9523,
+      "step": 539
+    },
+    {
+      "epoch": 0.5917808219178082,
+      "grad_norm": 2.088125705718994,
+      "learning_rate": 4.488220958570269e-06,
+      "loss": 1.9271,
+      "step": 540
+    },
+    {
+      "epoch": 0.5917808219178082,
+      "eval_loss": 2.134958505630493,
+      "eval_runtime": 36.8342,
+      "eval_samples_per_second": 27.149,
+      "eval_steps_per_second": 1.14,
+      "step": 540
+    },
+    {
+      "epoch": 0.5928767123287672,
+      "grad_norm": 2.1423559188842773,
+      "learning_rate": 4.486190089358245e-06,
+      "loss": 1.9467,
+      "step": 541
+    },
+    {
+      "epoch": 0.5939726027397261,
+      "grad_norm": 2.2169737815856934,
+      "learning_rate": 4.484159220146223e-06,
+      "loss": 1.8901,
+      "step": 542
+    },
+    {
+      "epoch": 0.5950684931506849,
+      "grad_norm": 2.419947385787964,
+      "learning_rate": 4.4821283509342e-06,
+      "loss": 2.0064,
+      "step": 543
+    },
+    {
+      "epoch": 0.5961643835616438,
+      "grad_norm": 2.0396435260772705,
+      "learning_rate": 4.480097481722178e-06,
+      "loss": 1.9819,
+      "step": 544
+    },
+    {
+      "epoch": 0.5972602739726027,
+      "grad_norm": 2.210700511932373,
+      "learning_rate": 4.478066612510154e-06,
+      "loss": 1.9578,
+      "step": 545
+    },
+    {
+      "epoch": 0.5983561643835617,
+      "grad_norm": 2.159698009490967,
+      "learning_rate": 4.4760357432981325e-06,
+      "loss": 1.9313,
+      "step": 546
+    },
+    {
+      "epoch": 0.5994520547945206,
+      "grad_norm": 2.125058650970459,
+      "learning_rate": 4.474004874086109e-06,
+      "loss": 1.9549,
+      "step": 547
+    },
+    {
+      "epoch": 0.6005479452054795,
+      "grad_norm": 2.477320432662964,
+      "learning_rate": 4.471974004874087e-06,
+      "loss": 2.0883,
+      "step": 548
+    },
+    {
+      "epoch": 0.6016438356164383,
+      "grad_norm": 2.214901924133301,
+      "learning_rate": 4.4699431356620635e-06,
+      "loss": 1.9184,
+      "step": 549
+    },
+    {
+      "epoch": 0.6027397260273972,
+      "grad_norm": 2.1001312732696533,
+      "learning_rate": 4.467912266450041e-06,
+      "loss": 2.0231,
+      "step": 550
+    },
+    {
+      "epoch": 0.6027397260273972,
+      "eval_loss": 2.1315157413482666,
+      "eval_runtime": 36.8131,
+      "eval_samples_per_second": 27.164,
+      "eval_steps_per_second": 1.141,
+      "step": 550
+    },
+    {
+      "epoch": 0.6038356164383561,
+      "grad_norm": 2.1030538082122803,
+      "learning_rate": 4.465881397238018e-06,
+      "loss": 1.967,
+      "step": 551
+    },
+    {
+      "epoch": 0.6049315068493151,
+      "grad_norm": 2.0390465259552,
+      "learning_rate": 4.463850528025995e-06,
+      "loss": 1.9615,
+      "step": 552
+    },
+    {
+      "epoch": 0.606027397260274,
+      "grad_norm": 2.1331627368927,
+      "learning_rate": 4.4618196588139726e-06,
+      "loss": 1.9667,
+      "step": 553
+    },
+    {
+      "epoch": 0.6071232876712329,
+      "grad_norm": 2.1987671852111816,
+      "learning_rate": 4.45978878960195e-06,
+      "loss": 1.9922,
+      "step": 554
+    },
+    {
+      "epoch": 0.6082191780821918,
+      "grad_norm": 2.127434253692627,
+      "learning_rate": 4.457757920389927e-06,
+      "loss": 1.9913,
+      "step": 555
+    },
+    {
+      "epoch": 0.6093150684931506,
+      "grad_norm": 2.2486324310302734,
+      "learning_rate": 4.455727051177904e-06,
+      "loss": 1.9239,
+      "step": 556
+    },
+    {
+      "epoch": 0.6104109589041096,
+      "grad_norm": 2.127575397491455,
+      "learning_rate": 4.453696181965882e-06,
+      "loss": 1.9521,
+      "step": 557
+    },
+    {
+      "epoch": 0.6115068493150685,
+      "grad_norm": 2.0798254013061523,
+      "learning_rate": 4.451665312753859e-06,
+      "loss": 2.0085,
+      "step": 558
+    },
+    {
+      "epoch": 0.6126027397260274,
+      "grad_norm": 2.287574291229248,
+      "learning_rate": 4.449634443541836e-06,
+      "loss": 1.98,
+      "step": 559
+    },
+    {
+      "epoch": 0.6136986301369863,
+      "grad_norm": 2.2056541442871094,
+      "learning_rate": 4.4476035743298135e-06,
+      "loss": 2.0282,
+      "step": 560
+    },
+    {
+      "epoch": 0.6136986301369863,
+      "eval_loss": 2.1287026405334473,
+      "eval_runtime": 36.7788,
+      "eval_samples_per_second": 27.19,
+      "eval_steps_per_second": 1.142,
+      "step": 560
+    },
+    {
+      "epoch": 0.6147945205479453,
+      "grad_norm": 2.169157028198242,
+      "learning_rate": 4.445572705117791e-06,
+      "loss": 1.9308,
+      "step": 561
+    },
+    {
+      "epoch": 0.6158904109589041,
+      "grad_norm": 2.1035149097442627,
+      "learning_rate": 4.443541835905768e-06,
+      "loss": 1.9507,
+      "step": 562
+    },
+    {
+      "epoch": 0.616986301369863,
+      "grad_norm": 2.1174328327178955,
+      "learning_rate": 4.441510966693745e-06,
+      "loss": 2.0162,
+      "step": 563
+    },
+    {
+      "epoch": 0.6180821917808219,
+      "grad_norm": 2.201641321182251,
+      "learning_rate": 4.439480097481723e-06,
+      "loss": 1.9573,
+      "step": 564
+    },
+    {
+      "epoch": 0.6191780821917808,
+      "grad_norm": 2.1763603687286377,
+      "learning_rate": 4.4374492282697e-06,
+      "loss": 2.0149,
+      "step": 565
+    },
+    {
+      "epoch": 0.6202739726027398,
+      "grad_norm": 2.2866439819335938,
+      "learning_rate": 4.435418359057677e-06,
+      "loss": 1.8836,
+      "step": 566
+    },
+    {
+      "epoch": 0.6213698630136987,
+      "grad_norm": 2.2060160636901855,
+      "learning_rate": 4.433387489845654e-06,
+      "loss": 1.9806,
+      "step": 567
+    },
+    {
+      "epoch": 0.6224657534246575,
+      "grad_norm": 2.0614662170410156,
+      "learning_rate": 4.431356620633632e-06,
+      "loss": 1.9821,
+      "step": 568
+    },
+    {
+      "epoch": 0.6235616438356164,
+      "grad_norm": 2.1055402755737305,
+      "learning_rate": 4.429325751421609e-06,
+      "loss": 1.9365,
+      "step": 569
+    },
+    {
+      "epoch": 0.6246575342465753,
+      "grad_norm": 1.9907679557800293,
+      "learning_rate": 4.427294882209586e-06,
+      "loss": 1.9429,
+      "step": 570
+    },
+    {
+      "epoch": 0.6246575342465753,
+      "eval_loss": 2.1286206245422363,
+      "eval_runtime": 36.7748,
+      "eval_samples_per_second": 27.193,
+      "eval_steps_per_second": 1.142,
+      "step": 570
+    },
+    {
+      "epoch": 0.6257534246575343,
+      "grad_norm": 2.0169243812561035,
+      "learning_rate": 4.4252640129975635e-06,
+      "loss": 1.9643,
+      "step": 571
+    },
+    {
+      "epoch": 0.6268493150684932,
+      "grad_norm": 2.1082401275634766,
+      "learning_rate": 4.423233143785541e-06,
+      "loss": 1.9009,
+      "step": 572
+    },
+    {
+      "epoch": 0.6279452054794521,
+      "grad_norm": 2.202542543411255,
+      "learning_rate": 4.421202274573518e-06,
+      "loss": 1.9694,
+      "step": 573
+    },
+    {
+      "epoch": 0.6290410958904109,
+      "grad_norm": 2.028489828109741,
+      "learning_rate": 4.4191714053614945e-06,
+      "loss": 1.9697,
+      "step": 574
+    },
+    {
+      "epoch": 0.6301369863013698,
+      "grad_norm": 2.134274482727051,
+      "learning_rate": 4.417140536149473e-06,
+      "loss": 1.8672,
+      "step": 575
+    },
+    {
+      "epoch": 0.6312328767123287,
+      "grad_norm": 2.063542366027832,
+      "learning_rate": 4.415109666937449e-06,
+      "loss": 1.909,
+      "step": 576
+    },
+    {
+      "epoch": 0.6323287671232877,
+      "grad_norm": 1.9747039079666138,
+      "learning_rate": 4.413078797725427e-06,
+      "loss": 1.9795,
+      "step": 577
+    },
+    {
+      "epoch": 0.6334246575342466,
+      "grad_norm": 2.063673496246338,
+      "learning_rate": 4.411047928513404e-06,
+      "loss": 1.9283,
+      "step": 578
+    },
+    {
+      "epoch": 0.6345205479452055,
+      "grad_norm": 2.024611711502075,
+      "learning_rate": 4.409017059301382e-06,
+      "loss": 1.9682,
+      "step": 579
+    },
+    {
+      "epoch": 0.6356164383561644,
+      "grad_norm": 2.0471599102020264,
+      "learning_rate": 4.406986190089358e-06,
+      "loss": 1.969,
+      "step": 580
+    },
+    {
+      "epoch": 0.6356164383561644,
+      "eval_loss": 2.1233692169189453,
+      "eval_runtime": 36.8185,
+      "eval_samples_per_second": 27.16,
+      "eval_steps_per_second": 1.141,
+      "step": 580
+    },
+    {
+      "epoch": 0.6367123287671232,
+      "grad_norm": 2.1038553714752197,
+      "learning_rate": 4.404955320877336e-06,
+      "loss": 1.9915,
+      "step": 581
+    },
+    {
+      "epoch": 0.6378082191780822,
+      "grad_norm": 2.1222152709960938,
+      "learning_rate": 4.402924451665313e-06,
+      "loss": 1.9892,
+      "step": 582
+    },
+    {
+      "epoch": 0.6389041095890411,
+      "grad_norm": 2.09137225151062,
+      "learning_rate": 4.400893582453291e-06,
+      "loss": 1.934,
+      "step": 583
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 2.2158281803131104,
+      "learning_rate": 4.398862713241267e-06,
+      "loss": 1.9188,
+      "step": 584
+    },
+    {
+      "epoch": 0.6410958904109589,
+      "grad_norm": 2.34619140625,
+      "learning_rate": 4.396831844029245e-06,
+      "loss": 1.9495,
+      "step": 585
+    },
+    {
+      "epoch": 0.6421917808219179,
+      "grad_norm": 2.2732315063476562,
+      "learning_rate": 4.394800974817222e-06,
+      "loss": 1.9632,
+      "step": 586
+    },
+    {
+      "epoch": 0.6432876712328767,
+      "grad_norm": 2.153338670730591,
+      "learning_rate": 4.392770105605199e-06,
+      "loss": 1.9551,
+      "step": 587
+    },
+    {
+      "epoch": 0.6443835616438356,
+      "grad_norm": 2.2201521396636963,
+      "learning_rate": 4.390739236393176e-06,
+      "loss": 1.9677,
+      "step": 588
+    },
+    {
+      "epoch": 0.6454794520547945,
+      "grad_norm": 1.9894570112228394,
+      "learning_rate": 4.388708367181154e-06,
+      "loss": 1.9826,
+      "step": 589
+    },
+    {
+      "epoch": 0.6465753424657534,
+      "grad_norm": 2.145061492919922,
+      "learning_rate": 4.386677497969131e-06,
+      "loss": 1.9826,
+      "step": 590
+    },
+    {
+      "epoch": 0.6465753424657534,
+      "eval_loss": 2.122312545776367,
+      "eval_runtime": 37.1153,
+      "eval_samples_per_second": 26.943,
+      "eval_steps_per_second": 1.132,
+      "step": 590
+    },
+    {
+      "epoch": 0.6476712328767124,
+      "grad_norm": 2.257237672805786,
+      "learning_rate": 4.384646628757108e-06,
+      "loss": 1.9722,
+      "step": 591
+    },
+    {
+      "epoch": 0.6487671232876713,
+      "grad_norm": 2.2406375408172607,
+      "learning_rate": 4.3826157595450855e-06,
+      "loss": 2.0141,
+      "step": 592
+    },
+    {
+      "epoch": 0.6498630136986301,
+      "grad_norm": 1.9656853675842285,
+      "learning_rate": 4.380584890333063e-06,
+      "loss": 1.9435,
+      "step": 593
+    },
+    {
+      "epoch": 0.650958904109589,
+      "grad_norm": 2.1063120365142822,
+      "learning_rate": 4.37855402112104e-06,
+      "loss": 1.9001,
+      "step": 594
+    },
+    {
+      "epoch": 0.6520547945205479,
+      "grad_norm": 2.1594269275665283,
+      "learning_rate": 4.376523151909017e-06,
+      "loss": 1.8596,
+      "step": 595
+    },
+    {
+      "epoch": 0.6531506849315069,
+      "grad_norm": 2.2261908054351807,
+      "learning_rate": 4.3744922826969946e-06,
+      "loss": 2.016,
+      "step": 596
+    },
+    {
+      "epoch": 0.6542465753424658,
+      "grad_norm": 1.9940093755722046,
+      "learning_rate": 4.372461413484972e-06,
+      "loss": 1.9293,
+      "step": 597
+    },
+    {
+      "epoch": 0.6553424657534247,
+      "grad_norm": 2.62186861038208,
+      "learning_rate": 4.370430544272949e-06,
+      "loss": 1.9656,
+      "step": 598
+    },
+    {
+      "epoch": 0.6564383561643835,
+      "grad_norm": 2.3531494140625,
+      "learning_rate": 4.368399675060926e-06,
+      "loss": 1.9984,
+      "step": 599
+    },
+    {
+      "epoch": 0.6575342465753424,
+      "grad_norm": 2.362769365310669,
+      "learning_rate": 4.366368805848904e-06,
+      "loss": 1.9971,
+      "step": 600
+    },
+    {
+      "epoch": 0.6575342465753424,
+      "eval_loss": 2.1172940731048584,
+      "eval_runtime": 37.0498,
+      "eval_samples_per_second": 26.991,
+      "eval_steps_per_second": 1.134,
+      "step": 600
+    },
+    {
+      "epoch": 0.6586301369863014,
+      "grad_norm": 2.3806607723236084,
+      "learning_rate": 4.364337936636881e-06,
+      "loss": 1.9566,
+      "step": 601
+    },
+    {
+      "epoch": 0.6597260273972603,
+      "grad_norm": 2.5531814098358154,
+      "learning_rate": 4.362307067424858e-06,
+      "loss": 1.9002,
+      "step": 602
+    },
+    {
+      "epoch": 0.6608219178082192,
+      "grad_norm": 2.283364772796631,
+      "learning_rate": 4.3602761982128355e-06,
+      "loss": 1.9633,
+      "step": 603
+    },
+    {
+      "epoch": 0.6619178082191781,
+      "grad_norm": 2.180171489715576,
+      "learning_rate": 4.358245329000813e-06,
+      "loss": 1.9277,
+      "step": 604
+    },
+    {
+      "epoch": 0.663013698630137,
+      "grad_norm": 2.0771350860595703,
+      "learning_rate": 4.35621445978879e-06,
+      "loss": 1.9484,
+      "step": 605
+    },
+    {
+      "epoch": 0.6641095890410958,
+      "grad_norm": 2.3772811889648438,
+      "learning_rate": 4.354183590576767e-06,
+      "loss": 1.9564,
+      "step": 606
+    },
+    {
+      "epoch": 0.6652054794520548,
+      "grad_norm": 2.205486297607422,
+      "learning_rate": 4.352152721364745e-06,
+      "loss": 1.9389,
+      "step": 607
+    },
+    {
+      "epoch": 0.6663013698630137,
+      "grad_norm": 2.085190773010254,
+      "learning_rate": 4.350121852152722e-06,
+      "loss": 1.8948,
+      "step": 608
+    },
+    {
+      "epoch": 0.6673972602739726,
+      "grad_norm": 2.4398982524871826,
+      "learning_rate": 4.348090982940699e-06,
+      "loss": 1.9736,
+      "step": 609
+    },
+    {
+      "epoch": 0.6684931506849315,
+      "grad_norm": 2.194087028503418,
+      "learning_rate": 4.3460601137286764e-06,
+      "loss": 1.9607,
+      "step": 610
+    },
+    {
+      "epoch": 0.6684931506849315,
+      "eval_loss": 2.115041971206665,
+      "eval_runtime": 36.9738,
+      "eval_samples_per_second": 27.046,
+      "eval_steps_per_second": 1.136,
+      "step": 610
+    },
+    {
+      "epoch": 0.6695890410958905,
+      "grad_norm": 2.143054485321045,
+      "learning_rate": 4.344029244516654e-06,
+      "loss": 1.9733,
+      "step": 611
+    },
+    {
+      "epoch": 0.6706849315068493,
+      "grad_norm": 2.2484617233276367,
+      "learning_rate": 4.341998375304631e-06,
+      "loss": 1.9417,
+      "step": 612
+    },
+    {
+      "epoch": 0.6717808219178082,
+      "grad_norm": 2.0430805683135986,
+      "learning_rate": 4.339967506092607e-06,
+      "loss": 2.0192,
+      "step": 613
+    },
+    {
+      "epoch": 0.6728767123287671,
+      "grad_norm": 2.061089277267456,
+      "learning_rate": 4.3379366368805855e-06,
+      "loss": 1.9543,
+      "step": 614
+    },
+    {
+      "epoch": 0.673972602739726,
+      "grad_norm": 2.298022747039795,
+      "learning_rate": 4.335905767668562e-06,
+      "loss": 1.9579,
+      "step": 615
+    },
+    {
+      "epoch": 0.675068493150685,
+      "grad_norm": 2.1418962478637695,
+      "learning_rate": 4.33387489845654e-06,
+      "loss": 1.938,
+      "step": 616
+    },
+    {
+      "epoch": 0.6761643835616439,
+      "grad_norm": 2.1005501747131348,
+      "learning_rate": 4.3318440292445165e-06,
+      "loss": 1.972,
+      "step": 617
+    },
+    {
+      "epoch": 0.6772602739726027,
+      "grad_norm": 2.1665523052215576,
+      "learning_rate": 4.329813160032495e-06,
+      "loss": 1.9535,
+      "step": 618
+    },
+    {
+      "epoch": 0.6783561643835616,
+      "grad_norm": 2.144580841064453,
+      "learning_rate": 4.327782290820471e-06,
+      "loss": 1.9188,
+      "step": 619
+    },
+    {
+      "epoch": 0.6794520547945205,
+      "grad_norm": 1.9855908155441284,
+      "learning_rate": 4.325751421608449e-06,
+      "loss": 1.912,
+      "step": 620
+    },
+    {
+      "epoch": 0.6794520547945205,
+      "eval_loss": 2.118319272994995,
+      "eval_runtime": 36.9677,
+      "eval_samples_per_second": 27.051,
+      "eval_steps_per_second": 1.136,
+      "step": 620
+    },
+    {
+      "epoch": 0.6805479452054795,
+      "grad_norm": 2.395538091659546,
+      "learning_rate": 4.323720552396426e-06,
+      "loss": 1.9886,
+      "step": 621
+    },
+    {
+      "epoch": 0.6816438356164384,
+      "grad_norm": 2.190574884414673,
+      "learning_rate": 4.321689683184404e-06,
+      "loss": 1.9326,
+      "step": 622
+    },
+    {
+      "epoch": 0.6827397260273973,
+      "grad_norm": 2.1826560497283936,
+      "learning_rate": 4.31965881397238e-06,
+      "loss": 1.9278,
+      "step": 623
+    },
+    {
+      "epoch": 0.6838356164383562,
+      "grad_norm": 2.151097536087036,
+      "learning_rate": 4.3176279447603574e-06,
+      "loss": 1.9359,
+      "step": 624
+    },
+    {
+      "epoch": 0.684931506849315,
+      "grad_norm": 2.1830122470855713,
+      "learning_rate": 4.315597075548335e-06,
+      "loss": 1.9728,
+      "step": 625
+    },
+    {
+      "epoch": 0.686027397260274,
+      "grad_norm": 2.203325033187866,
+      "learning_rate": 4.313566206336312e-06,
+      "loss": 1.9638,
+      "step": 626
+    },
+    {
+      "epoch": 0.6871232876712329,
+      "grad_norm": 2.7240309715270996,
+      "learning_rate": 4.311535337124289e-06,
+      "loss": 1.9907,
+      "step": 627
+    },
+    {
+      "epoch": 0.6882191780821918,
+      "grad_norm": 2.133599281311035,
+      "learning_rate": 4.3095044679122665e-06,
+      "loss": 1.9352,
+      "step": 628
+    },
+    {
+      "epoch": 0.6893150684931507,
+      "grad_norm": 2.0318551063537598,
+      "learning_rate": 4.307473598700244e-06,
+      "loss": 1.9432,
+      "step": 629
+    },
+    {
+      "epoch": 0.6904109589041096,
+      "grad_norm": 2.502101182937622,
+      "learning_rate": 4.305442729488221e-06,
+      "loss": 1.9641,
+      "step": 630
+    },
+    {
+      "epoch": 0.6904109589041096,
+      "eval_loss": 2.1158926486968994,
+      "eval_runtime": 36.8818,
+      "eval_samples_per_second": 27.114,
+      "eval_steps_per_second": 1.139,
+      "step": 630
+    },
+    {
+      "epoch": 0.6915068493150685,
+      "grad_norm": 2.3620944023132324,
+      "learning_rate": 4.303411860276198e-06,
+      "loss": 1.8841,
+      "step": 631
+    },
+    {
+      "epoch": 0.6926027397260274,
+      "grad_norm": 2.3139240741729736,
+      "learning_rate": 4.301380991064176e-06,
+      "loss": 1.9896,
+      "step": 632
+    },
+    {
+      "epoch": 0.6936986301369863,
+      "grad_norm": 2.314877510070801,
+      "learning_rate": 4.299350121852153e-06,
+      "loss": 1.9655,
+      "step": 633
+    },
+    {
+      "epoch": 0.6947945205479452,
+      "grad_norm": 2.036371946334839,
+      "learning_rate": 4.29731925264013e-06,
+      "loss": 1.9917,
+      "step": 634
+    },
+    {
+      "epoch": 0.6958904109589041,
+      "grad_norm": 2.1859662532806396,
+      "learning_rate": 4.2952883834281075e-06,
+      "loss": 1.9805,
+      "step": 635
+    },
+    {
+      "epoch": 0.6969863013698631,
+      "grad_norm": 2.2317097187042236,
+      "learning_rate": 4.293257514216085e-06,
+      "loss": 1.9402,
+      "step": 636
+    },
+    {
+      "epoch": 0.6980821917808219,
+      "grad_norm": 2.3249905109405518,
+      "learning_rate": 4.291226645004062e-06,
+      "loss": 1.9013,
+      "step": 637
+    },
+    {
+      "epoch": 0.6991780821917808,
+      "grad_norm": 2.199758768081665,
+      "learning_rate": 4.289195775792039e-06,
+      "loss": 1.9783,
+      "step": 638
+    },
+    {
+      "epoch": 0.7002739726027397,
+      "grad_norm": 2.3598549365997314,
+      "learning_rate": 4.287164906580017e-06,
+      "loss": 1.9936,
+      "step": 639
+    },
+    {
+      "epoch": 0.7013698630136986,
+      "grad_norm": 2.1705522537231445,
+      "learning_rate": 4.285134037367994e-06,
+      "loss": 1.9304,
+      "step": 640
+    },
+    {
+      "epoch": 0.7013698630136986,
+      "eval_loss": 2.1055190563201904,
+      "eval_runtime": 36.8673,
+      "eval_samples_per_second": 27.124,
+      "eval_steps_per_second": 1.139,
+      "step": 640
+    },
+    {
+      "epoch": 0.7024657534246576,
+      "grad_norm": 2.3253612518310547,
+      "learning_rate": 4.283103168155971e-06,
+      "loss": 1.9196,
+      "step": 641
+    },
+    {
+      "epoch": 0.7035616438356165,
+      "grad_norm": 2.3054087162017822,
+      "learning_rate": 4.281072298943948e-06,
+      "loss": 1.9053,
+      "step": 642
+    },
+    {
+      "epoch": 0.7046575342465753,
+      "grad_norm": 2.262057304382324,
+      "learning_rate": 4.279041429731926e-06,
+      "loss": 1.8999,
+      "step": 643
+    },
+    {
+      "epoch": 0.7057534246575342,
+      "grad_norm": 2.421391248703003,
+      "learning_rate": 4.277010560519903e-06,
+      "loss": 1.9438,
+      "step": 644
+    },
+    {
+      "epoch": 0.7068493150684931,
+      "grad_norm": 2.2841498851776123,
+      "learning_rate": 4.27497969130788e-06,
+      "loss": 1.9187,
+      "step": 645
+    },
+    {
+      "epoch": 0.707945205479452,
+      "grad_norm": 2.3226938247680664,
+      "learning_rate": 4.2729488220958575e-06,
+      "loss": 1.9275,
+      "step": 646
+    },
+    {
+      "epoch": 0.709041095890411,
+      "grad_norm": 2.2306301593780518,
+      "learning_rate": 4.270917952883835e-06,
+      "loss": 1.9684,
+      "step": 647
+    },
+    {
+      "epoch": 0.7101369863013699,
+      "grad_norm": 2.0551652908325195,
+      "learning_rate": 4.268887083671812e-06,
+      "loss": 1.9849,
+      "step": 648
+    },
+    {
+      "epoch": 0.7112328767123288,
+      "grad_norm": 2.2952210903167725,
+      "learning_rate": 4.266856214459789e-06,
+      "loss": 1.9026,
+      "step": 649
+    },
+    {
+      "epoch": 0.7123287671232876,
+      "grad_norm": 2.1630797386169434,
+      "learning_rate": 4.264825345247766e-06,
+      "loss": 2.0184,
+      "step": 650
+    },
+    {
+      "epoch": 0.7123287671232876,
+      "eval_loss": 2.1037871837615967,
+      "eval_runtime": 36.8221,
+      "eval_samples_per_second": 27.158,
+      "eval_steps_per_second": 1.141,
+      "step": 650
+    },
+    {
+      "epoch": 0.7134246575342466,
+      "grad_norm": 1.964150309562683,
+      "learning_rate": 4.262794476035744e-06,
+      "loss": 1.8932,
+      "step": 651
+    },
+    {
+      "epoch": 0.7145205479452055,
+      "grad_norm": 2.1474685668945312,
+      "learning_rate": 4.26076360682372e-06,
+      "loss": 1.9383,
+      "step": 652
+    },
+    {
+      "epoch": 0.7156164383561644,
+      "grad_norm": 2.199824094772339,
+      "learning_rate": 4.2587327376116984e-06,
+      "loss": 1.8599,
+      "step": 653
+    },
+    {
+      "epoch": 0.7167123287671233,
+      "grad_norm": 2.12156343460083,
+      "learning_rate": 4.256701868399675e-06,
+      "loss": 1.9162,
+      "step": 654
+    },
+    {
+      "epoch": 0.7178082191780822,
+      "grad_norm": 2.3832154273986816,
+      "learning_rate": 4.254670999187653e-06,
+      "loss": 1.9613,
+      "step": 655
+    },
+    {
+      "epoch": 0.718904109589041,
+      "grad_norm": 2.063906192779541,
+      "learning_rate": 4.252640129975629e-06,
+      "loss": 1.9421,
+      "step": 656
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 2.184842586517334,
+      "learning_rate": 4.2506092607636075e-06,
+      "loss": 1.9471,
+      "step": 657
+    },
+    {
+      "epoch": 0.7210958904109589,
+      "grad_norm": 2.1340036392211914,
+      "learning_rate": 4.248578391551584e-06,
+      "loss": 1.931,
+      "step": 658
+    },
+    {
+      "epoch": 0.7221917808219178,
+      "grad_norm": 2.117544651031494,
+      "learning_rate": 4.246547522339562e-06,
+      "loss": 1.8698,
+      "step": 659
+    },
+    {
+      "epoch": 0.7232876712328767,
+      "grad_norm": 2.308344841003418,
+      "learning_rate": 4.2445166531275385e-06,
+      "loss": 1.9217,
+      "step": 660
+    },
+    {
+      "epoch": 0.7232876712328767,
+      "eval_loss": 2.1086323261260986,
+      "eval_runtime": 36.8082,
+      "eval_samples_per_second": 27.168,
+      "eval_steps_per_second": 1.141,
+      "step": 660
+    },
+    {
+      "epoch": 0.7243835616438357,
+      "grad_norm": 2.2239017486572266,
+      "learning_rate": 4.242485783915517e-06,
+      "loss": 1.9286,
+      "step": 661
+    },
+    {
+      "epoch": 0.7254794520547945,
+      "grad_norm": 2.263697624206543,
+      "learning_rate": 4.240454914703493e-06,
+      "loss": 1.8821,
+      "step": 662
+    },
+    {
+      "epoch": 0.7265753424657534,
+      "grad_norm": 2.4305059909820557,
+      "learning_rate": 4.23842404549147e-06,
+      "loss": 1.8996,
+      "step": 663
+    },
+    {
+      "epoch": 0.7276712328767123,
+      "grad_norm": 2.2333695888519287,
+      "learning_rate": 4.236393176279448e-06,
+      "loss": 1.8906,
+      "step": 664
+    },
+    {
+      "epoch": 0.7287671232876712,
+      "grad_norm": 2.160982608795166,
+      "learning_rate": 4.234362307067425e-06,
+      "loss": 1.879,
+      "step": 665
+    },
+    {
+      "epoch": 0.7298630136986302,
+      "grad_norm": 2.3707094192504883,
+      "learning_rate": 4.232331437855402e-06,
+      "loss": 1.9577,
+      "step": 666
+    },
+    {
+      "epoch": 0.7309589041095891,
+      "grad_norm": 2.0502827167510986,
+      "learning_rate": 4.2303005686433795e-06,
+      "loss": 1.9801,
+      "step": 667
+    },
+    {
+      "epoch": 0.7320547945205479,
+      "grad_norm": 2.0648176670074463,
+      "learning_rate": 4.228269699431357e-06,
+      "loss": 1.978,
+      "step": 668
+    },
+    {
+      "epoch": 0.7331506849315068,
+      "grad_norm": 2.346316337585449,
+      "learning_rate": 4.226238830219334e-06,
+      "loss": 1.9387,
+      "step": 669
+    },
+    {
+      "epoch": 0.7342465753424657,
+      "grad_norm": 1.9734236001968384,
+      "learning_rate": 4.224207961007311e-06,
+      "loss": 1.9396,
+      "step": 670
+    },
+    {
+      "epoch": 0.7342465753424657,
+      "eval_loss": 2.1104705333709717,
+      "eval_runtime": 36.8144,
+      "eval_samples_per_second": 27.163,
+      "eval_steps_per_second": 1.141,
+      "step": 670
+    },
+    {
+      "epoch": 0.7353424657534247,
+      "grad_norm": 2.3170549869537354,
+      "learning_rate": 4.2221770917952886e-06,
+      "loss": 1.9398,
+      "step": 671
+    },
+    {
+      "epoch": 0.7364383561643836,
+      "grad_norm": 2.0437264442443848,
+      "learning_rate": 4.220146222583266e-06,
+      "loss": 1.9284,
+      "step": 672
+    },
+    {
+      "epoch": 0.7375342465753425,
+      "grad_norm": 2.0808510780334473,
+      "learning_rate": 4.218115353371243e-06,
+      "loss": 1.9425,
+      "step": 673
+    },
+    {
+      "epoch": 0.7386301369863014,
+      "grad_norm": 2.0854742527008057,
+      "learning_rate": 4.21608448415922e-06,
+      "loss": 1.9358,
+      "step": 674
+    },
+    {
+      "epoch": 0.7397260273972602,
+      "grad_norm": 2.131622552871704,
+      "learning_rate": 4.214053614947198e-06,
+      "loss": 1.9048,
+      "step": 675
+    },
+    {
+      "epoch": 0.7408219178082192,
+      "grad_norm": 2.276359796524048,
+      "learning_rate": 4.212022745735175e-06,
+      "loss": 1.9871,
+      "step": 676
+    },
+    {
+      "epoch": 0.7419178082191781,
+      "grad_norm": 2.2194812297821045,
+      "learning_rate": 4.209991876523152e-06,
+      "loss": 1.882,
+      "step": 677
+    },
+    {
+      "epoch": 0.743013698630137,
+      "grad_norm": 2.181891679763794,
+      "learning_rate": 4.2079610073111295e-06,
+      "loss": 1.9546,
+      "step": 678
+    },
+    {
+      "epoch": 0.7441095890410959,
+      "grad_norm": 2.0956673622131348,
+      "learning_rate": 4.205930138099107e-06,
+      "loss": 1.8836,
+      "step": 679
+    },
+    {
+      "epoch": 0.7452054794520548,
+      "grad_norm": 2.057894229888916,
+      "learning_rate": 4.203899268887084e-06,
+      "loss": 1.9842,
+      "step": 680
+    },
+    {
+      "epoch": 0.7452054794520548,
+      "eval_loss": 2.104182720184326,
+      "eval_runtime": 36.8189,
+      "eval_samples_per_second": 27.16,
+      "eval_steps_per_second": 1.141,
+      "step": 680
+    },
+    {
+      "epoch": 0.7463013698630137,
+      "grad_norm": 2.0965662002563477,
+      "learning_rate": 4.201868399675061e-06,
+      "loss": 1.9378,
+      "step": 681
+    },
+    {
+      "epoch": 0.7473972602739726,
+      "grad_norm": 2.1843457221984863,
+      "learning_rate": 4.199837530463039e-06,
+      "loss": 1.9784,
+      "step": 682
+    },
+    {
+      "epoch": 0.7484931506849315,
+      "grad_norm": 2.2044246196746826,
+      "learning_rate": 4.197806661251016e-06,
+      "loss": 1.8642,
+      "step": 683
+    },
+    {
+      "epoch": 0.7495890410958904,
+      "grad_norm": 2.202115058898926,
+      "learning_rate": 4.195775792038993e-06,
+      "loss": 1.929,
+      "step": 684
+    },
+    {
+      "epoch": 0.7506849315068493,
+      "grad_norm": 2.3671462535858154,
+      "learning_rate": 4.19374492282697e-06,
+      "loss": 1.9172,
+      "step": 685
+    },
+    {
+      "epoch": 0.7517808219178083,
+      "grad_norm": 2.0602803230285645,
+      "learning_rate": 4.191714053614948e-06,
+      "loss": 1.8503,
+      "step": 686
+    },
+    {
+      "epoch": 0.7528767123287671,
+      "grad_norm": 2.12145733833313,
+      "learning_rate": 4.189683184402924e-06,
+      "loss": 1.9511,
+      "step": 687
+    },
+    {
+      "epoch": 0.753972602739726,
+      "grad_norm": 2.1049914360046387,
+      "learning_rate": 4.187652315190902e-06,
+      "loss": 1.9043,
+      "step": 688
+    },
+    {
+      "epoch": 0.7550684931506849,
+      "grad_norm": 2.235497236251831,
+      "learning_rate": 4.185621445978879e-06,
+      "loss": 1.9784,
+      "step": 689
+    },
+    {
+      "epoch": 0.7561643835616438,
+      "grad_norm": 2.173222303390503,
+      "learning_rate": 4.183590576766857e-06,
+      "loss": 1.9052,
+      "step": 690
+    },
+    {
+      "epoch": 0.7561643835616438,
+      "eval_loss": 2.0982251167297363,
+      "eval_runtime": 37.1446,
+      "eval_samples_per_second": 26.922,
+      "eval_steps_per_second": 1.131,
+      "step": 690
+    },
+    {
+      "epoch": 0.7572602739726028,
+      "grad_norm": 2.0693562030792236,
+      "learning_rate": 4.181559707554833e-06,
+      "loss": 1.9223,
+      "step": 691
+    },
+    {
+      "epoch": 0.7583561643835617,
+      "grad_norm": 2.209419012069702,
+      "learning_rate": 4.179528838342811e-06,
+      "loss": 1.9709,
+      "step": 692
+    },
+    {
+      "epoch": 0.7594520547945205,
+      "grad_norm": 2.066041946411133,
+      "learning_rate": 4.177497969130788e-06,
+      "loss": 1.889,
+      "step": 693
+    },
+    {
+      "epoch": 0.7605479452054794,
+      "grad_norm": 2.4114325046539307,
+      "learning_rate": 4.175467099918766e-06,
+      "loss": 1.9059,
+      "step": 694
+    },
+    {
+      "epoch": 0.7616438356164383,
+      "grad_norm": 2.0185043811798096,
+      "learning_rate": 4.173436230706742e-06,
+      "loss": 1.9877,
+      "step": 695
+    },
+    {
+      "epoch": 0.7627397260273973,
+      "grad_norm": 2.23392915725708,
+      "learning_rate": 4.1714053614947205e-06,
+      "loss": 1.9199,
+      "step": 696
+    },
+    {
+      "epoch": 0.7638356164383562,
+      "grad_norm": 2.2158737182617188,
+      "learning_rate": 4.169374492282697e-06,
+      "loss": 1.9001,
+      "step": 697
+    },
+    {
+      "epoch": 0.7649315068493151,
+      "grad_norm": 2.1511690616607666,
+      "learning_rate": 4.167343623070675e-06,
+      "loss": 1.9166,
+      "step": 698
+    },
+    {
+      "epoch": 0.766027397260274,
+      "grad_norm": 2.374741554260254,
+      "learning_rate": 4.165312753858652e-06,
+      "loss": 1.9684,
+      "step": 699
+    },
+    {
+      "epoch": 0.7671232876712328,
+      "grad_norm": 2.0409998893737793,
+      "learning_rate": 4.163281884646629e-06,
+      "loss": 1.9177,
+      "step": 700
+    },
+    {
+      "epoch": 0.7671232876712328,
+      "eval_loss": 2.096895933151245,
+      "eval_runtime": 37.066,
+      "eval_samples_per_second": 26.979,
+      "eval_steps_per_second": 1.133,
+      "step": 700
+    },
+    {
+      "epoch": 0.7682191780821918,
+      "grad_norm": 2.3173422813415527,
+      "learning_rate": 4.161251015434607e-06,
+      "loss": 1.9071,
+      "step": 701
+    },
+    {
+      "epoch": 0.7693150684931507,
+      "grad_norm": 2.077842950820923,
+      "learning_rate": 4.159220146222583e-06,
+      "loss": 1.935,
+      "step": 702
+    },
+    {
+      "epoch": 0.7704109589041096,
+      "grad_norm": 2.0518617630004883,
+      "learning_rate": 4.157189277010561e-06,
+      "loss": 1.8942,
+      "step": 703
+    },
+    {
+      "epoch": 0.7715068493150685,
+      "grad_norm": 1.9950882196426392,
+      "learning_rate": 4.155158407798538e-06,
+      "loss": 1.917,
+      "step": 704
+    },
+    {
+      "epoch": 0.7726027397260274,
+      "grad_norm": 2.3222219944000244,
+      "learning_rate": 4.153127538586516e-06,
+      "loss": 1.9551,
+      "step": 705
+    },
+    {
+      "epoch": 0.7736986301369863,
+      "grad_norm": 2.129333972930908,
+      "learning_rate": 4.151096669374492e-06,
+      "loss": 1.9204,
+      "step": 706
+    },
+    {
+      "epoch": 0.7747945205479452,
+      "grad_norm": 2.0357553958892822,
+      "learning_rate": 4.1490658001624705e-06,
+      "loss": 1.9399,
+      "step": 707
+    },
+    {
+      "epoch": 0.7758904109589041,
+      "grad_norm": 2.233694553375244,
+      "learning_rate": 4.147034930950447e-06,
+      "loss": 1.9182,
+      "step": 708
+    },
+    {
+      "epoch": 0.776986301369863,
+      "grad_norm": 2.1391239166259766,
+      "learning_rate": 4.145004061738425e-06,
+      "loss": 1.9157,
+      "step": 709
+    },
+    {
+      "epoch": 0.7780821917808219,
+      "grad_norm": 2.162320613861084,
+      "learning_rate": 4.1429731925264015e-06,
+      "loss": 1.9115,
+      "step": 710
+    },
+    {
+      "epoch": 0.7780821917808219,
+      "eval_loss": 2.0954039096832275,
+      "eval_runtime": 36.9862,
+      "eval_samples_per_second": 27.037,
+      "eval_steps_per_second": 1.136,
+      "step": 710
+    },
+    {
+      "epoch": 0.7791780821917809,
+      "grad_norm": 2.2382194995880127,
+      "learning_rate": 4.140942323314379e-06,
+      "loss": 1.8428,
+      "step": 711
+    },
+    {
+      "epoch": 0.7802739726027397,
+      "grad_norm": 2.3286185264587402,
+      "learning_rate": 4.138911454102356e-06,
+      "loss": 1.8906,
+      "step": 712
+    },
+    {
+      "epoch": 0.7813698630136986,
+      "grad_norm": 2.182708501815796,
+      "learning_rate": 4.136880584890333e-06,
+      "loss": 1.9132,
+      "step": 713
+    },
+    {
+      "epoch": 0.7824657534246575,
+      "grad_norm": 1.9978337287902832,
+      "learning_rate": 4.1348497156783106e-06,
+      "loss": 1.9433,
+      "step": 714
+    },
+    {
+      "epoch": 0.7835616438356164,
+      "grad_norm": 2.3092565536499023,
+      "learning_rate": 4.132818846466288e-06,
+      "loss": 1.9374,
+      "step": 715
+    },
+    {
+      "epoch": 0.7846575342465754,
+      "grad_norm": 2.0351054668426514,
+      "learning_rate": 4.130787977254265e-06,
+      "loss": 1.9361,
+      "step": 716
+    },
+    {
+      "epoch": 0.7857534246575343,
+      "grad_norm": 2.14770770072937,
+      "learning_rate": 4.128757108042242e-06,
+      "loss": 1.9553,
+      "step": 717
+    },
+    {
+      "epoch": 0.7868493150684932,
+      "grad_norm": 2.0613555908203125,
+      "learning_rate": 4.12672623883022e-06,
+      "loss": 1.9506,
+      "step": 718
+    },
+    {
+      "epoch": 0.787945205479452,
+      "grad_norm": 2.1312081813812256,
+      "learning_rate": 4.124695369618197e-06,
+      "loss": 2.0079,
+      "step": 719
+    },
+    {
+      "epoch": 0.7890410958904109,
+      "grad_norm": 2.088552474975586,
+      "learning_rate": 4.122664500406174e-06,
+      "loss": 1.9166,
+      "step": 720
+    },
+    {
+      "epoch": 0.7890410958904109,
+      "eval_loss": 2.093032121658325,
+      "eval_runtime": 36.8821,
+      "eval_samples_per_second": 27.113,
+      "eval_steps_per_second": 1.139,
+      "step": 720
+    },
+    {
+      "epoch": 0.7901369863013699,
+      "grad_norm": 2.1748929023742676,
+      "learning_rate": 4.1206336311941515e-06,
+      "loss": 1.9212,
+      "step": 721
+    },
+    {
+      "epoch": 0.7912328767123288,
+      "grad_norm": 2.1492016315460205,
+      "learning_rate": 4.118602761982129e-06,
+      "loss": 1.9452,
+      "step": 722
+    },
+    {
+      "epoch": 0.7923287671232877,
+      "grad_norm": 2.3364970684051514,
+      "learning_rate": 4.116571892770106e-06,
+      "loss": 1.8821,
+      "step": 723
+    },
+    {
+      "epoch": 0.7934246575342466,
+      "grad_norm": 2.318488597869873,
+      "learning_rate": 4.114541023558083e-06,
+      "loss": 1.8597,
+      "step": 724
+    },
+    {
+      "epoch": 0.7945205479452054,
+      "grad_norm": 2.153841257095337,
+      "learning_rate": 4.112510154346061e-06,
+      "loss": 1.9843,
+      "step": 725
+    },
+    {
+      "epoch": 0.7956164383561644,
+      "grad_norm": 2.3464651107788086,
+      "learning_rate": 4.110479285134038e-06,
+      "loss": 1.9105,
+      "step": 726
+    },
+    {
+      "epoch": 0.7967123287671233,
+      "grad_norm": 2.34413743019104,
+      "learning_rate": 4.108448415922015e-06,
+      "loss": 1.9285,
+      "step": 727
+    },
+    {
+      "epoch": 0.7978082191780822,
+      "grad_norm": 2.0822196006774902,
+      "learning_rate": 4.1064175467099924e-06,
+      "loss": 1.9536,
+      "step": 728
+    },
+    {
+      "epoch": 0.7989041095890411,
+      "grad_norm": 2.1978065967559814,
+      "learning_rate": 4.10438667749797e-06,
+      "loss": 1.9487,
+      "step": 729
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 2.4218783378601074,
+      "learning_rate": 4.102355808285947e-06,
+      "loss": 1.8113,
+      "step": 730
+    },
+    {
+      "epoch": 0.8,
+      "eval_loss": 2.0963101387023926,
+      "eval_runtime": 36.8853,
+      "eval_samples_per_second": 27.111,
+      "eval_steps_per_second": 1.139,
+      "step": 730
+    },
+    {
+      "epoch": 0.8010958904109589,
+      "grad_norm": 2.3887739181518555,
+      "learning_rate": 4.100324939073924e-06,
+      "loss": 2.0024,
+      "step": 731
+    },
+    {
+      "epoch": 0.8021917808219178,
+      "grad_norm": 2.296579360961914,
+      "learning_rate": 4.0982940698619015e-06,
+      "loss": 1.9092,
+      "step": 732
+    },
+    {
+      "epoch": 0.8032876712328767,
+      "grad_norm": 2.3881773948669434,
+      "learning_rate": 4.096263200649879e-06,
+      "loss": 1.9112,
+      "step": 733
+    },
+    {
+      "epoch": 0.8043835616438356,
+      "grad_norm": 2.40120530128479,
+      "learning_rate": 4.094232331437856e-06,
+      "loss": 1.9191,
+      "step": 734
+    },
+    {
+      "epoch": 0.8054794520547945,
+      "grad_norm": 2.184706211090088,
+      "learning_rate": 4.092201462225833e-06,
+      "loss": 1.8857,
+      "step": 735
+    },
+    {
+      "epoch": 0.8065753424657535,
+      "grad_norm": 2.3338732719421387,
+      "learning_rate": 4.090170593013811e-06,
+      "loss": 1.8889,
+      "step": 736
+    },
+    {
+      "epoch": 0.8076712328767123,
+      "grad_norm": 2.4065468311309814,
+      "learning_rate": 4.088139723801787e-06,
+      "loss": 1.9309,
+      "step": 737
+    },
+    {
+      "epoch": 0.8087671232876712,
+      "grad_norm": 2.044992446899414,
+      "learning_rate": 4.086108854589765e-06,
+      "loss": 1.8727,
+      "step": 738
+    },
+    {
+      "epoch": 0.8098630136986301,
+      "grad_norm": 2.23860239982605,
+      "learning_rate": 4.084077985377742e-06,
+      "loss": 1.8936,
+      "step": 739
+    },
+    {
+      "epoch": 0.810958904109589,
+      "grad_norm": 2.1324410438537598,
+      "learning_rate": 4.08204711616572e-06,
+      "loss": 1.9035,
+      "step": 740
+    },
+    {
+      "epoch": 0.810958904109589,
+      "eval_loss": 2.092885732650757,
+      "eval_runtime": 36.7991,
+      "eval_samples_per_second": 27.175,
+      "eval_steps_per_second": 1.141,
+      "step": 740
+    },
+    {
+      "epoch": 0.812054794520548,
+      "grad_norm": 2.337340831756592,
+      "learning_rate": 4.080016246953696e-06,
+      "loss": 1.9747,
+      "step": 741
+    },
+    {
+      "epoch": 0.8131506849315069,
+      "grad_norm": 2.262801170349121,
+      "learning_rate": 4.077985377741674e-06,
+      "loss": 1.8817,
+      "step": 742
+    },
+    {
+      "epoch": 0.8142465753424658,
+      "grad_norm": 2.0494258403778076,
+      "learning_rate": 4.075954508529651e-06,
+      "loss": 1.9493,
+      "step": 743
+    },
+    {
+      "epoch": 0.8153424657534246,
+      "grad_norm": 2.085829019546509,
+      "learning_rate": 4.073923639317629e-06,
+      "loss": 1.9611,
+      "step": 744
+    },
+    {
+      "epoch": 0.8164383561643835,
+      "grad_norm": 2.105806827545166,
+      "learning_rate": 4.071892770105605e-06,
+      "loss": 1.9779,
+      "step": 745
+    },
+    {
+      "epoch": 0.8175342465753425,
+      "grad_norm": 2.2664732933044434,
+      "learning_rate": 4.069861900893583e-06,
+      "loss": 1.9009,
+      "step": 746
+    },
+    {
+      "epoch": 0.8186301369863014,
+      "grad_norm": 1.9899768829345703,
+      "learning_rate": 4.06783103168156e-06,
+      "loss": 1.8585,
+      "step": 747
+    },
+    {
+      "epoch": 0.8197260273972603,
+      "grad_norm": 2.242236614227295,
+      "learning_rate": 4.065800162469538e-06,
+      "loss": 1.8904,
+      "step": 748
+    },
+    {
+      "epoch": 0.8208219178082192,
+      "grad_norm": 2.4481139183044434,
+      "learning_rate": 4.063769293257514e-06,
+      "loss": 1.8922,
+      "step": 749
+    },
+    {
+      "epoch": 0.821917808219178,
+      "grad_norm": 2.231722354888916,
+      "learning_rate": 4.061738424045492e-06,
+      "loss": 1.9292,
+      "step": 750
+    },
+    {
+      "epoch": 0.821917808219178,
+      "eval_loss": 2.0909852981567383,
+      "eval_runtime": 36.8255,
+      "eval_samples_per_second": 27.155,
+      "eval_steps_per_second": 1.141,
+      "step": 750
+    },
+    {
+      "epoch": 0.823013698630137,
+      "grad_norm": 2.323070764541626,
+      "learning_rate": 4.059707554833469e-06,
+      "loss": 1.9053,
+      "step": 751
+    },
+    {
+      "epoch": 0.8241095890410959,
+      "grad_norm": 2.137535572052002,
+      "learning_rate": 4.057676685621446e-06,
+      "loss": 1.9272,
+      "step": 752
+    },
+    {
+      "epoch": 0.8252054794520548,
+      "grad_norm": 2.2087643146514893,
+      "learning_rate": 4.0556458164094235e-06,
+      "loss": 1.9541,
+      "step": 753
+    },
+    {
+      "epoch": 0.8263013698630137,
+      "grad_norm": 2.04811692237854,
+      "learning_rate": 4.053614947197401e-06,
+      "loss": 1.922,
+      "step": 754
+    },
+    {
+      "epoch": 0.8273972602739726,
+      "grad_norm": 2.062622308731079,
+      "learning_rate": 4.051584077985378e-06,
+      "loss": 1.8388,
+      "step": 755
+    },
+    {
+      "epoch": 0.8284931506849315,
+      "grad_norm": 2.14631986618042,
+      "learning_rate": 4.049553208773355e-06,
+      "loss": 1.9153,
+      "step": 756
+    },
+    {
+      "epoch": 0.8295890410958904,
+      "grad_norm": 2.353529214859009,
+      "learning_rate": 4.047522339561333e-06,
+      "loss": 1.9623,
+      "step": 757
+    },
+    {
+      "epoch": 0.8306849315068493,
+      "grad_norm": 2.122342348098755,
+      "learning_rate": 4.04549147034931e-06,
+      "loss": 1.9095,
+      "step": 758
+    },
+    {
+      "epoch": 0.8317808219178082,
+      "grad_norm": 2.2540230751037598,
+      "learning_rate": 4.043460601137287e-06,
+      "loss": 1.9068,
+      "step": 759
+    },
+    {
+      "epoch": 0.8328767123287671,
+      "grad_norm": 2.164384365081787,
+      "learning_rate": 4.041429731925264e-06,
+      "loss": 2.0239,
+      "step": 760
+    },
+    {
+      "epoch": 0.8328767123287671,
+      "eval_loss": 2.0904648303985596,
+      "eval_runtime": 36.7617,
+      "eval_samples_per_second": 27.202,
+      "eval_steps_per_second": 1.142,
+      "step": 760
+    },
+    {
+      "epoch": 0.8339726027397261,
+      "grad_norm": 2.2487905025482178,
+      "learning_rate": 4.039398862713242e-06,
+      "loss": 1.9109,
+      "step": 761
+    },
+    {
+      "epoch": 0.8350684931506849,
+      "grad_norm": 2.0947930812835693,
+      "learning_rate": 4.037367993501219e-06,
+      "loss": 1.9133,
+      "step": 762
+    },
+    {
+      "epoch": 0.8361643835616438,
+      "grad_norm": 2.2259862422943115,
+      "learning_rate": 4.035337124289196e-06,
+      "loss": 1.9821,
+      "step": 763
+    },
+    {
+      "epoch": 0.8372602739726027,
+      "grad_norm": 2.0616455078125,
+      "learning_rate": 4.0333062550771735e-06,
+      "loss": 1.9037,
+      "step": 764
+    },
+    {
+      "epoch": 0.8383561643835616,
+      "grad_norm": 2.2704474925994873,
+      "learning_rate": 4.031275385865151e-06,
+      "loss": 1.8664,
+      "step": 765
+    },
+    {
+      "epoch": 0.8394520547945206,
+      "grad_norm": 2.0059332847595215,
+      "learning_rate": 4.029244516653128e-06,
+      "loss": 1.908,
+      "step": 766
+    },
+    {
+      "epoch": 0.8405479452054795,
+      "grad_norm": 2.0462095737457275,
+      "learning_rate": 4.027213647441105e-06,
+      "loss": 1.9397,
+      "step": 767
+    },
+    {
+      "epoch": 0.8416438356164384,
+      "grad_norm": 2.150207996368408,
+      "learning_rate": 4.025182778229083e-06,
+      "loss": 1.8505,
+      "step": 768
+    },
+    {
+      "epoch": 0.8427397260273972,
+      "grad_norm": 2.174863815307617,
+      "learning_rate": 4.02315190901706e-06,
+      "loss": 1.8895,
+      "step": 769
+    },
+    {
+      "epoch": 0.8438356164383561,
+      "grad_norm": 2.1907355785369873,
+      "learning_rate": 4.021121039805037e-06,
+      "loss": 1.9156,
+      "step": 770
+    },
+    {
+      "epoch": 0.8438356164383561,
+      "eval_loss": 2.0900955200195312,
+      "eval_runtime": 36.8004,
+      "eval_samples_per_second": 27.174,
+      "eval_steps_per_second": 1.141,
+      "step": 770
+    },
+    {
+      "epoch": 0.8449315068493151,
+      "grad_norm": 2.2831835746765137,
+      "learning_rate": 4.0190901705930144e-06,
+      "loss": 1.9115,
+      "step": 771
+    },
+    {
+      "epoch": 0.846027397260274,
+      "grad_norm": 2.039520263671875,
+      "learning_rate": 4.017059301380992e-06,
+      "loss": 2.0047,
+      "step": 772
+    },
+    {
+      "epoch": 0.8471232876712329,
+      "grad_norm": 2.147209405899048,
+      "learning_rate": 4.015028432168969e-06,
+      "loss": 1.8845,
+      "step": 773
+    },
+    {
+      "epoch": 0.8482191780821918,
+      "grad_norm": 2.168997287750244,
+      "learning_rate": 4.012997562956945e-06,
+      "loss": 1.9606,
+      "step": 774
+    },
+    {
+      "epoch": 0.8493150684931506,
+      "grad_norm": 2.038966417312622,
+      "learning_rate": 4.0109666937449235e-06,
+      "loss": 1.8734,
+      "step": 775
+    },
+    {
+      "epoch": 0.8504109589041096,
+      "grad_norm": 2.0531768798828125,
+      "learning_rate": 4.0089358245329e-06,
+      "loss": 1.889,
+      "step": 776
+    },
+    {
+      "epoch": 0.8515068493150685,
+      "grad_norm": 2.348008155822754,
+      "learning_rate": 4.006904955320878e-06,
+      "loss": 1.8972,
+      "step": 777
+    },
+    {
+      "epoch": 0.8526027397260274,
+      "grad_norm": 2.1647253036499023,
+      "learning_rate": 4.0048740861088545e-06,
+      "loss": 1.8652,
+      "step": 778
+    },
+    {
+      "epoch": 0.8536986301369863,
+      "grad_norm": 2.1971640586853027,
+      "learning_rate": 4.002843216896833e-06,
+      "loss": 1.9495,
+      "step": 779
+    },
+    {
+      "epoch": 0.8547945205479452,
+      "grad_norm": 2.169632911682129,
+      "learning_rate": 4.000812347684809e-06,
+      "loss": 1.9307,
+      "step": 780
+    },
+    {
+      "epoch": 0.8547945205479452,
+      "eval_loss": 2.087860107421875,
+      "eval_runtime": 36.7982,
+      "eval_samples_per_second": 27.175,
+      "eval_steps_per_second": 1.141,
+      "step": 780
+    },
+    {
+      "epoch": 0.8558904109589041,
+      "grad_norm": 2.4424400329589844,
+      "learning_rate": 3.998781478472787e-06,
+      "loss": 1.8849,
+      "step": 781
+    },
+    {
+      "epoch": 0.856986301369863,
+      "grad_norm": 2.2226672172546387,
+      "learning_rate": 3.996750609260764e-06,
+      "loss": 1.8975,
+      "step": 782
+    },
+    {
+      "epoch": 0.8580821917808219,
+      "grad_norm": 2.1898491382598877,
+      "learning_rate": 3.994719740048742e-06,
+      "loss": 1.8751,
+      "step": 783
+    },
+    {
+      "epoch": 0.8591780821917808,
+      "grad_norm": 2.466963052749634,
+      "learning_rate": 3.992688870836718e-06,
+      "loss": 2.0155,
+      "step": 784
+    },
+    {
+      "epoch": 0.8602739726027397,
+      "grad_norm": 2.184039831161499,
+      "learning_rate": 3.990658001624696e-06,
+      "loss": 1.9133,
+      "step": 785
+    },
+    {
+      "epoch": 0.8613698630136987,
+      "grad_norm": 2.089228868484497,
+      "learning_rate": 3.988627132412673e-06,
+      "loss": 1.8707,
+      "step": 786
+    },
+    {
+      "epoch": 0.8624657534246576,
+      "grad_norm": 2.0700666904449463,
+      "learning_rate": 3.98659626320065e-06,
+      "loss": 1.9487,
+      "step": 787
+    },
+    {
+      "epoch": 0.8635616438356164,
+      "grad_norm": 2.1495554447174072,
+      "learning_rate": 3.984565393988627e-06,
+      "loss": 1.9037,
+      "step": 788
+    },
+    {
+      "epoch": 0.8646575342465753,
+      "grad_norm": 2.0916988849639893,
+      "learning_rate": 3.9825345247766046e-06,
+      "loss": 1.9096,
+      "step": 789
+    },
+    {
+      "epoch": 0.8657534246575342,
+      "grad_norm": 2.072387933731079,
+      "learning_rate": 3.980503655564582e-06,
+      "loss": 1.9139,
+      "step": 790
+    },
+    {
+      "epoch": 0.8657534246575342,
+      "eval_loss": 2.0876142978668213,
+      "eval_runtime": 37.1068,
+      "eval_samples_per_second": 26.949,
+      "eval_steps_per_second": 1.132,
+      "step": 790
+    },
+    {
+      "epoch": 0.8668493150684932,
+      "grad_norm": 2.1206343173980713,
+      "learning_rate": 3.978472786352559e-06,
+      "loss": 1.9331,
+      "step": 791
+    },
+    {
+      "epoch": 0.8679452054794521,
+      "grad_norm": 2.20263934135437,
+      "learning_rate": 3.976441917140536e-06,
+      "loss": 1.9297,
+      "step": 792
+    },
+    {
+      "epoch": 0.869041095890411,
+      "grad_norm": 2.027554750442505,
+      "learning_rate": 3.974411047928514e-06,
+      "loss": 1.919,
+      "step": 793
+    },
+    {
+      "epoch": 0.8701369863013698,
+      "grad_norm": 2.0291616916656494,
+      "learning_rate": 3.972380178716491e-06,
+      "loss": 1.9682,
+      "step": 794
+    },
+    {
+      "epoch": 0.8712328767123287,
+      "grad_norm": 2.2719762325286865,
+      "learning_rate": 3.970349309504468e-06,
+      "loss": 1.9407,
+      "step": 795
+    },
+    {
+      "epoch": 0.8723287671232877,
+      "grad_norm": 2.0452375411987305,
+      "learning_rate": 3.9683184402924455e-06,
+      "loss": 1.9487,
+      "step": 796
+    },
+    {
+      "epoch": 0.8734246575342466,
+      "grad_norm": 2.170441150665283,
+      "learning_rate": 3.966287571080423e-06,
+      "loss": 1.8651,
+      "step": 797
+    },
+    {
+      "epoch": 0.8745205479452055,
+      "grad_norm": 2.3370516300201416,
+      "learning_rate": 3.9642567018684e-06,
+      "loss": 1.8599,
+      "step": 798
+    },
+    {
+      "epoch": 0.8756164383561644,
+      "grad_norm": 2.336737632751465,
+      "learning_rate": 3.962225832656377e-06,
+      "loss": 1.9304,
+      "step": 799
+    },
+    {
+      "epoch": 0.8767123287671232,
+      "grad_norm": 1.9708369970321655,
+      "learning_rate": 3.960194963444355e-06,
+      "loss": 1.9125,
+      "step": 800
+    },
+    {
+      "epoch": 0.8767123287671232,
+      "eval_loss": 2.0812273025512695,
+      "eval_runtime": 37.0752,
+      "eval_samples_per_second": 26.972,
+      "eval_steps_per_second": 1.133,
+      "step": 800
+    },
+    {
+      "epoch": 0.8778082191780822,
+      "grad_norm": 2.154219627380371,
+      "learning_rate": 3.958164094232332e-06,
+      "loss": 1.8304,
+      "step": 801
+    },
+    {
+      "epoch": 0.8789041095890411,
+      "grad_norm": 2.237786054611206,
+      "learning_rate": 3.956133225020309e-06,
+      "loss": 1.9147,
+      "step": 802
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 2.07143235206604,
+      "learning_rate": 3.954102355808286e-06,
+      "loss": 1.9128,
+      "step": 803
+    },
+    {
+      "epoch": 0.8810958904109589,
+      "grad_norm": 2.2780392169952393,
+      "learning_rate": 3.952071486596264e-06,
+      "loss": 1.9145,
+      "step": 804
+    },
+    {
+      "epoch": 0.8821917808219178,
+      "grad_norm": 2.2396292686462402,
+      "learning_rate": 3.950040617384241e-06,
+      "loss": 1.9049,
+      "step": 805
+    },
+    {
+      "epoch": 0.8832876712328767,
+      "grad_norm": 2.426511287689209,
+      "learning_rate": 3.948009748172218e-06,
+      "loss": 1.8984,
+      "step": 806
+    },
+    {
+      "epoch": 0.8843835616438356,
+      "grad_norm": 2.1874325275421143,
+      "learning_rate": 3.9459788789601955e-06,
+      "loss": 1.8888,
+      "step": 807
+    },
+    {
+      "epoch": 0.8854794520547945,
+      "grad_norm": 2.038098096847534,
+      "learning_rate": 3.943948009748173e-06,
+      "loss": 1.9582,
+      "step": 808
+    },
+    {
+      "epoch": 0.8865753424657534,
+      "grad_norm": 2.3748130798339844,
+      "learning_rate": 3.94191714053615e-06,
+      "loss": 1.9091,
+      "step": 809
+    },
+    {
+      "epoch": 0.8876712328767123,
+      "grad_norm": 2.2457494735717773,
+      "learning_rate": 3.939886271324127e-06,
+      "loss": 1.9067,
+      "step": 810
+    },
+    {
+      "epoch": 0.8876712328767123,
+      "eval_loss": 2.0766079425811768,
+      "eval_runtime": 36.9896,
+      "eval_samples_per_second": 27.035,
+      "eval_steps_per_second": 1.135,
+      "step": 810
+    },
+    {
+      "epoch": 0.8887671232876713,
+      "grad_norm": 2.156644821166992,
+      "learning_rate": 3.937855402112104e-06,
+      "loss": 1.9347,
+      "step": 811
+    },
+    {
+      "epoch": 0.8898630136986302,
+      "grad_norm": 2.3991472721099854,
+      "learning_rate": 3.935824532900082e-06,
+      "loss": 1.8645,
+      "step": 812
+    },
+    {
+      "epoch": 0.890958904109589,
+      "grad_norm": 2.252729654312134,
+      "learning_rate": 3.933793663688058e-06,
+      "loss": 1.8537,
+      "step": 813
+    },
+    {
+      "epoch": 0.8920547945205479,
+      "grad_norm": 2.2917964458465576,
+      "learning_rate": 3.9317627944760365e-06,
+      "loss": 1.9271,
+      "step": 814
+    },
+    {
+      "epoch": 0.8931506849315068,
+      "grad_norm": 2.217310667037964,
+      "learning_rate": 3.929731925264013e-06,
+      "loss": 1.9406,
+      "step": 815
+    },
+    {
+      "epoch": 0.8942465753424658,
+      "grad_norm": 2.167931079864502,
+      "learning_rate": 3.927701056051991e-06,
+      "loss": 1.8554,
+      "step": 816
+    },
+    {
+      "epoch": 0.8953424657534247,
+      "grad_norm": 2.18337082862854,
+      "learning_rate": 3.9256701868399674e-06,
+      "loss": 1.8795,
+      "step": 817
+    },
+    {
+      "epoch": 0.8964383561643836,
+      "grad_norm": 2.2348973751068115,
+      "learning_rate": 3.9236393176279456e-06,
+      "loss": 1.8831,
+      "step": 818
+    },
+    {
+      "epoch": 0.8975342465753424,
+      "grad_norm": 2.1822657585144043,
+      "learning_rate": 3.921608448415922e-06,
+      "loss": 1.8728,
+      "step": 819
+    },
+    {
+      "epoch": 0.8986301369863013,
+      "grad_norm": 2.5253243446350098,
+      "learning_rate": 3.9195775792039e-06,
+      "loss": 1.8633,
+      "step": 820
+    },
+    {
+      "epoch": 0.8986301369863013,
+      "eval_loss": 2.0769834518432617,
+      "eval_runtime": 36.838,
+      "eval_samples_per_second": 27.146,
+      "eval_steps_per_second": 1.14,
+      "step": 820
+    },
+    {
+      "epoch": 0.8997260273972603,
+      "grad_norm": 2.2109973430633545,
+      "learning_rate": 3.9175467099918765e-06,
+      "loss": 1.912,
+      "step": 821
+    },
+    {
+      "epoch": 0.9008219178082192,
+      "grad_norm": 2.053692102432251,
+      "learning_rate": 3.915515840779855e-06,
+      "loss": 1.8928,
+      "step": 822
+    },
+    {
+      "epoch": 0.9019178082191781,
+      "grad_norm": 2.1418213844299316,
+      "learning_rate": 3.913484971567831e-06,
+      "loss": 1.9399,
+      "step": 823
+    },
+    {
+      "epoch": 0.903013698630137,
+      "grad_norm": 2.3615458011627197,
+      "learning_rate": 3.911454102355808e-06,
+      "loss": 1.9293,
+      "step": 824
+    },
+    {
+      "epoch": 0.9041095890410958,
+      "grad_norm": 2.098254680633545,
+      "learning_rate": 3.909423233143786e-06,
+      "loss": 1.8301,
+      "step": 825
+    },
+    {
+      "epoch": 0.9052054794520548,
+      "grad_norm": 2.4473679065704346,
+      "learning_rate": 3.907392363931763e-06,
+      "loss": 1.9254,
+      "step": 826
+    },
+    {
+      "epoch": 0.9063013698630137,
+      "grad_norm": 2.252843141555786,
+      "learning_rate": 3.90536149471974e-06,
+      "loss": 1.9155,
+      "step": 827
+    },
+    {
+      "epoch": 0.9073972602739726,
+      "grad_norm": 2.1574337482452393,
+      "learning_rate": 3.9033306255077175e-06,
+      "loss": 1.9809,
+      "step": 828
+    },
+    {
+      "epoch": 0.9084931506849315,
+      "grad_norm": 2.2899720668792725,
+      "learning_rate": 3.901299756295695e-06,
+      "loss": 2.0045,
+      "step": 829
+    },
+    {
+      "epoch": 0.9095890410958904,
+      "grad_norm": 2.227285146713257,
+      "learning_rate": 3.899268887083672e-06,
+      "loss": 2.026,
+      "step": 830
+    },
+    {
+      "epoch": 0.9095890410958904,
+      "eval_loss": 2.0764760971069336,
+      "eval_runtime": 36.9195,
+      "eval_samples_per_second": 27.086,
+      "eval_steps_per_second": 1.138,
+      "step": 830
+    },
+    {
+      "epoch": 0.9106849315068493,
+      "grad_norm": 2.1935312747955322,
+      "learning_rate": 3.897238017871649e-06,
+      "loss": 1.8922,
+      "step": 831
+    },
+    {
+      "epoch": 0.9117808219178082,
+      "grad_norm": 2.129692316055298,
+      "learning_rate": 3.8952071486596266e-06,
+      "loss": 1.8981,
+      "step": 832
+    },
+    {
+      "epoch": 0.9128767123287671,
+      "grad_norm": 2.4435646533966064,
+      "learning_rate": 3.893176279447604e-06,
+      "loss": 1.91,
+      "step": 833
+    },
+    {
+      "epoch": 0.913972602739726,
+      "grad_norm": 2.1556732654571533,
+      "learning_rate": 3.891145410235581e-06,
+      "loss": 1.8688,
+      "step": 834
+    },
+    {
+      "epoch": 0.915068493150685,
+      "grad_norm": 2.175891160964966,
+      "learning_rate": 3.889114541023558e-06,
+      "loss": 1.9777,
+      "step": 835
+    },
+    {
+      "epoch": 0.9161643835616439,
+      "grad_norm": 2.220466375350952,
+      "learning_rate": 3.887083671811536e-06,
+      "loss": 1.9152,
+      "step": 836
+    },
+    {
+      "epoch": 0.9172602739726028,
+      "grad_norm": 2.331397294998169,
+      "learning_rate": 3.885052802599513e-06,
+      "loss": 1.892,
+      "step": 837
+    },
+    {
+      "epoch": 0.9183561643835616,
+      "grad_norm": 2.2434773445129395,
+      "learning_rate": 3.88302193338749e-06,
+      "loss": 1.841,
+      "step": 838
+    },
+    {
+      "epoch": 0.9194520547945205,
+      "grad_norm": 2.3248002529144287,
+      "learning_rate": 3.8809910641754675e-06,
+      "loss": 1.9657,
+      "step": 839
+    },
+    {
+      "epoch": 0.9205479452054794,
+      "grad_norm": 2.090252637863159,
+      "learning_rate": 3.878960194963445e-06,
+      "loss": 1.9121,
+      "step": 840
+    },
+    {
+      "epoch": 0.9205479452054794,
+      "eval_loss": 2.0798065662384033,
+      "eval_runtime": 36.8557,
+      "eval_samples_per_second": 27.133,
+      "eval_steps_per_second": 1.14,
+      "step": 840
+    },
+    {
+      "epoch": 0.9216438356164384,
+      "grad_norm": 2.6968672275543213,
+      "learning_rate": 3.876929325751422e-06,
+      "loss": 1.9003,
+      "step": 841
+    },
+    {
+      "epoch": 0.9227397260273973,
+      "grad_norm": 2.356321096420288,
+      "learning_rate": 3.874898456539399e-06,
+      "loss": 1.9471,
+      "step": 842
+    },
+    {
+      "epoch": 0.9238356164383562,
+      "grad_norm": 2.2678606510162354,
+      "learning_rate": 3.872867587327377e-06,
+      "loss": 1.8488,
+      "step": 843
+    },
+    {
+      "epoch": 0.924931506849315,
+      "grad_norm": 2.1122310161590576,
+      "learning_rate": 3.870836718115354e-06,
+      "loss": 1.9247,
+      "step": 844
+    },
+    {
+      "epoch": 0.9260273972602739,
+      "grad_norm": 2.1219136714935303,
+      "learning_rate": 3.868805848903331e-06,
+      "loss": 1.8829,
+      "step": 845
+    },
+    {
+      "epoch": 0.9271232876712329,
+      "grad_norm": 2.175607204437256,
+      "learning_rate": 3.8667749796913084e-06,
+      "loss": 1.887,
+      "step": 846
+    },
+    {
+      "epoch": 0.9282191780821918,
+      "grad_norm": 2.156214714050293,
+      "learning_rate": 3.864744110479286e-06,
+      "loss": 1.9119,
+      "step": 847
+    },
+    {
+      "epoch": 0.9293150684931507,
+      "grad_norm": 2.158461570739746,
+      "learning_rate": 3.862713241267262e-06,
+      "loss": 1.9092,
+      "step": 848
+    },
+    {
+      "epoch": 0.9304109589041096,
+      "grad_norm": 2.180551767349243,
+      "learning_rate": 3.86068237205524e-06,
+      "loss": 1.946,
+      "step": 849
+    },
+    {
+      "epoch": 0.9315068493150684,
+      "grad_norm": 2.1030542850494385,
+      "learning_rate": 3.858651502843217e-06,
+      "loss": 1.9484,
+      "step": 850
+    },
+    {
+      "epoch": 0.9315068493150684,
+      "eval_loss": 2.0767791271209717,
+      "eval_runtime": 36.8047,
+      "eval_samples_per_second": 27.17,
+      "eval_steps_per_second": 1.141,
+      "step": 850
+    },
+    {
+      "epoch": 0.9326027397260274,
+      "grad_norm": 2.240769386291504,
+      "learning_rate": 3.856620633631195e-06,
+      "loss": 1.9114,
+      "step": 851
+    },
+    {
+      "epoch": 0.9336986301369863,
+      "grad_norm": 2.1964619159698486,
+      "learning_rate": 3.854589764419171e-06,
+      "loss": 1.8339,
+      "step": 852
+    },
+    {
+      "epoch": 0.9347945205479452,
+      "grad_norm": 2.2875845432281494,
+      "learning_rate": 3.852558895207149e-06,
+      "loss": 1.8942,
+      "step": 853
+    },
+    {
+      "epoch": 0.9358904109589041,
+      "grad_norm": 2.1462507247924805,
+      "learning_rate": 3.850528025995126e-06,
+      "loss": 1.9838,
+      "step": 854
+    },
+    {
+      "epoch": 0.936986301369863,
+      "grad_norm": 2.0982913970947266,
+      "learning_rate": 3.848497156783104e-06,
+      "loss": 1.9351,
+      "step": 855
+    },
+    {
+      "epoch": 0.938082191780822,
+      "grad_norm": 2.260263442993164,
+      "learning_rate": 3.84646628757108e-06,
+      "loss": 1.8745,
+      "step": 856
+    },
+    {
+      "epoch": 0.9391780821917808,
+      "grad_norm": 2.091778039932251,
+      "learning_rate": 3.8444354183590585e-06,
+      "loss": 1.8888,
+      "step": 857
+    },
+    {
+      "epoch": 0.9402739726027397,
+      "grad_norm": 2.054974317550659,
+      "learning_rate": 3.842404549147035e-06,
+      "loss": 1.8801,
+      "step": 858
+    },
+    {
+      "epoch": 0.9413698630136986,
+      "grad_norm": 2.079641580581665,
+      "learning_rate": 3.840373679935013e-06,
+      "loss": 1.8879,
+      "step": 859
+    },
+    {
+      "epoch": 0.9424657534246575,
+      "grad_norm": 2.3441810607910156,
+      "learning_rate": 3.8383428107229894e-06,
+      "loss": 1.8571,
+      "step": 860
+    },
+    {
+      "epoch": 0.9424657534246575,
+      "eval_loss": 2.0766854286193848,
+      "eval_runtime": 36.8414,
+      "eval_samples_per_second": 27.143,
+      "eval_steps_per_second": 1.14,
+      "step": 860
+    },
+    {
+      "epoch": 0.9435616438356165,
+      "grad_norm": 2.123061180114746,
+      "learning_rate": 3.836311941510967e-06,
+      "loss": 1.8922,
+      "step": 861
+    },
+    {
+      "epoch": 0.9446575342465754,
+      "grad_norm": 2.1573386192321777,
+      "learning_rate": 3.834281072298944e-06,
+      "loss": 1.8713,
+      "step": 862
+    },
+    {
+      "epoch": 0.9457534246575342,
+      "grad_norm": 2.225186824798584,
+      "learning_rate": 3.832250203086921e-06,
+      "loss": 1.925,
+      "step": 863
+    },
+    {
+      "epoch": 0.9468493150684931,
+      "grad_norm": 2.1539998054504395,
+      "learning_rate": 3.8302193338748985e-06,
+      "loss": 1.8721,
+      "step": 864
+    },
+    {
+      "epoch": 0.947945205479452,
+      "grad_norm": 2.075580358505249,
+      "learning_rate": 3.828188464662876e-06,
+      "loss": 1.9489,
+      "step": 865
+    },
+    {
+      "epoch": 0.949041095890411,
+      "grad_norm": 2.1789908409118652,
+      "learning_rate": 3.826157595450853e-06,
+      "loss": 1.9883,
+      "step": 866
+    },
+    {
+      "epoch": 0.9501369863013699,
+      "grad_norm": 2.0838615894317627,
+      "learning_rate": 3.82412672623883e-06,
+      "loss": 1.9149,
+      "step": 867
+    },
+    {
+      "epoch": 0.9512328767123288,
+      "grad_norm": 2.1321213245391846,
+      "learning_rate": 3.822095857026808e-06,
+      "loss": 1.8505,
+      "step": 868
+    },
+    {
+      "epoch": 0.9523287671232876,
+      "grad_norm": 2.1419479846954346,
+      "learning_rate": 3.820064987814785e-06,
+      "loss": 1.9226,
+      "step": 869
+    },
+    {
+      "epoch": 0.9534246575342465,
+      "grad_norm": 2.290163040161133,
+      "learning_rate": 3.818034118602762e-06,
+      "loss": 1.8737,
+      "step": 870
+    },
+    {
+      "epoch": 0.9534246575342465,
+      "eval_loss": 2.073598623275757,
+      "eval_runtime": 36.7896,
+      "eval_samples_per_second": 27.182,
+      "eval_steps_per_second": 1.142,
+      "step": 870
+    },
+    {
+      "epoch": 0.9545205479452055,
+      "grad_norm": 2.1951074600219727,
+      "learning_rate": 3.8160032493907395e-06,
+      "loss": 1.8699,
+      "step": 871
+    },
+    {
+      "epoch": 0.9556164383561644,
+      "grad_norm": 2.25960111618042,
+      "learning_rate": 3.813972380178717e-06,
+      "loss": 1.8813,
+      "step": 872
+    },
+    {
+      "epoch": 0.9567123287671233,
+      "grad_norm": 2.4376907348632812,
+      "learning_rate": 3.811941510966694e-06,
+      "loss": 1.8782,
+      "step": 873
+    },
+    {
+      "epoch": 0.9578082191780822,
+      "grad_norm": 1.980772614479065,
+      "learning_rate": 3.809910641754671e-06,
+      "loss": 1.9205,
+      "step": 874
+    },
+    {
+      "epoch": 0.958904109589041,
+      "grad_norm": 2.0699028968811035,
+      "learning_rate": 3.8078797725426486e-06,
+      "loss": 1.8674,
+      "step": 875
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 2.2765867710113525,
+      "learning_rate": 3.8058489033306254e-06,
+      "loss": 1.81,
+      "step": 876
+    },
+    {
+      "epoch": 0.9610958904109589,
+      "grad_norm": 2.0998799800872803,
+      "learning_rate": 3.803818034118603e-06,
+      "loss": 1.9061,
+      "step": 877
+    },
+    {
+      "epoch": 0.9621917808219178,
+      "grad_norm": 2.2362914085388184,
+      "learning_rate": 3.80178716490658e-06,
+      "loss": 1.8798,
+      "step": 878
+    },
+    {
+      "epoch": 0.9632876712328767,
+      "grad_norm": 2.0789835453033447,
+      "learning_rate": 3.7997562956945577e-06,
+      "loss": 1.9491,
+      "step": 879
+    },
+    {
+      "epoch": 0.9643835616438357,
+      "grad_norm": 2.0458688735961914,
+      "learning_rate": 3.7977254264825345e-06,
+      "loss": 1.931,
+      "step": 880
+    },
+    {
+      "epoch": 0.9643835616438357,
+      "eval_loss": 2.0672221183776855,
+      "eval_runtime": 36.8218,
+      "eval_samples_per_second": 27.158,
+      "eval_steps_per_second": 1.141,
+      "step": 880
+    },
+    {
+      "epoch": 0.9654794520547946,
+      "grad_norm": 2.361626625061035,
+      "learning_rate": 3.7956945572705122e-06,
+      "loss": 1.8815,
+      "step": 881
+    },
+    {
+      "epoch": 0.9665753424657534,
+      "grad_norm": 2.294160842895508,
+      "learning_rate": 3.793663688058489e-06,
+      "loss": 1.8649,
+      "step": 882
+    },
+    {
+      "epoch": 0.9676712328767123,
+      "grad_norm": 2.2242612838745117,
+      "learning_rate": 3.791632818846467e-06,
+      "loss": 1.9535,
+      "step": 883
+    },
+    {
+      "epoch": 0.9687671232876712,
+      "grad_norm": 2.220269203186035,
+      "learning_rate": 3.7896019496344436e-06,
+      "loss": 1.9506,
+      "step": 884
+    },
+    {
+      "epoch": 0.9698630136986301,
+      "grad_norm": 2.112518072128296,
+      "learning_rate": 3.787571080422421e-06,
+      "loss": 1.9487,
+      "step": 885
+    },
+    {
+      "epoch": 0.9709589041095891,
+      "grad_norm": 2.17695951461792,
+      "learning_rate": 3.785540211210398e-06,
+      "loss": 1.9467,
+      "step": 886
+    },
+    {
+      "epoch": 0.972054794520548,
+      "grad_norm": 2.0990567207336426,
+      "learning_rate": 3.7835093419983755e-06,
+      "loss": 1.9216,
+      "step": 887
+    },
+    {
+      "epoch": 0.9731506849315068,
+      "grad_norm": 2.222566604614258,
+      "learning_rate": 3.7814784727863527e-06,
+      "loss": 1.9043,
+      "step": 888
+    },
+    {
+      "epoch": 0.9742465753424657,
+      "grad_norm": 2.09890079498291,
+      "learning_rate": 3.77944760357433e-06,
+      "loss": 1.884,
+      "step": 889
+    },
+    {
+      "epoch": 0.9753424657534246,
+      "grad_norm": 2.136141061782837,
+      "learning_rate": 3.7774167343623073e-06,
+      "loss": 2.0032,
+      "step": 890
+    },
+    {
+      "epoch": 0.9753424657534246,
+      "eval_loss": 2.0665438175201416,
+      "eval_runtime": 37.1207,
+      "eval_samples_per_second": 26.939,
+      "eval_steps_per_second": 1.131,
+      "step": 890
+    },
+    {
+      "epoch": 0.9764383561643836,
+      "grad_norm": 2.267444372177124,
+      "learning_rate": 3.7753858651502846e-06,
+      "loss": 1.947,
+      "step": 891
+    },
+    {
+      "epoch": 0.9775342465753425,
+      "grad_norm": 2.1141815185546875,
+      "learning_rate": 3.773354995938262e-06,
+      "loss": 1.8705,
+      "step": 892
+    },
+    {
+      "epoch": 0.9786301369863014,
+      "grad_norm": 2.1036782264709473,
+      "learning_rate": 3.771324126726239e-06,
+      "loss": 1.9381,
+      "step": 893
+    },
+    {
+      "epoch": 0.9797260273972602,
+      "grad_norm": 2.081982135772705,
+      "learning_rate": 3.7692932575142164e-06,
+      "loss": 1.9079,
+      "step": 894
+    },
+    {
+      "epoch": 0.9808219178082191,
+      "grad_norm": 2.0815446376800537,
+      "learning_rate": 3.7672623883021937e-06,
+      "loss": 1.9417,
+      "step": 895
+    },
+    {
+      "epoch": 0.9819178082191781,
+      "grad_norm": 2.0531511306762695,
+      "learning_rate": 3.765231519090171e-06,
+      "loss": 1.876,
+      "step": 896
+    },
+    {
+      "epoch": 0.983013698630137,
+      "grad_norm": 2.212940216064453,
+      "learning_rate": 3.7632006498781482e-06,
+      "loss": 1.8008,
+      "step": 897
+    },
+    {
+      "epoch": 0.9841095890410959,
+      "grad_norm": 2.1541531085968018,
+      "learning_rate": 3.761169780666125e-06,
+      "loss": 1.9039,
+      "step": 898
+    },
+    {
+      "epoch": 0.9852054794520548,
+      "grad_norm": 2.113884925842285,
+      "learning_rate": 3.7591389114541028e-06,
+      "loss": 1.8562,
+      "step": 899
+    },
+    {
+      "epoch": 0.9863013698630136,
+      "grad_norm": 2.1503701210021973,
+      "learning_rate": 3.7571080422420796e-06,
+      "loss": 1.9052,
+      "step": 900
+    },
+    {
+      "epoch": 0.9863013698630136,
+      "eval_loss": 2.0664408206939697,
+      "eval_runtime": 37.0876,
+      "eval_samples_per_second": 26.963,
+      "eval_steps_per_second": 1.132,
+      "step": 900
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 2736,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.805578468425728e+17,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/training_args.bin b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..b7adbe5a2dbf27f05befec6cc65fbc93e90eefd0
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa882279512a8c278376c0c601206933f3c5662ddcd3c13c53bf32ae3680d14
+size 6011
diff --git a/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/zero_to_fp32.py b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoints/Llama-3.2-3B/babylm_shuffle_deterministic84_10M_seed0/runs/checkpoint-900/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)