add weights

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "/home/slseanwu/11891_codegen/air2/runs/deepseek-1.3b-full-arith-hard-PT-promptC12-interm-bs32-30K-240416/checkpoint-20000",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
+  "rope_theta": 100000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 32256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc0e472ea796a4b0e3ba96f79740fb6f66fd0ccf1d32f212f9853fecd655ba44
+size 4986380064

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:299028d5a9b41e2203b779cbc459e20821401373c9c1988bcd73bed7552c2c64
+size 399532808

model.safetensors.index.json

@@ -0,0 +1,226 @@
+{
+  "metadata": {
+    "total_size": 5385887744
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe34b13dee7d9da42d001abf122dbaecf50b6bf7f52cb9d0a5447bdaa240d3a5
+size 2699039674

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:183fdf51dbe79231d6619388215f2ac4056fa6643f2257500e69174f47856ec1
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f728bf1bb63de4031722624d28ecc479007645e4c48e28a89d409237d3a10ed4
+size 1064

trainer_state.json

@@ -0,0 +1,1119 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 2500,
+  "global_step": 15000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 1.482173204421997,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.4165,
+      "step": 100
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 1.0766079425811768,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.0351,
+      "step": 200
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.8878026008605957,
+      "learning_rate": 1.2e-05,
+      "loss": 0.9911,
+      "step": 300
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.8634878396987915,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.9833,
+      "step": 400
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.8714269399642944,
+      "learning_rate": 2e-05,
+      "loss": 0.9708,
+      "step": 500
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 1.1247708797454834,
+      "learning_rate": 1.9862068965517244e-05,
+      "loss": 0.9742,
+      "step": 600
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.8334652781486511,
+      "learning_rate": 1.9724137931034483e-05,
+      "loss": 0.9529,
+      "step": 700
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.8450727462768555,
+      "learning_rate": 1.9586206896551725e-05,
+      "loss": 0.9545,
+      "step": 800
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.9118452072143555,
+      "learning_rate": 1.9448275862068968e-05,
+      "loss": 0.9456,
+      "step": 900
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.7823046445846558,
+      "learning_rate": 1.931034482758621e-05,
+      "loss": 0.9338,
+      "step": 1000
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 1.1707388162612915,
+      "learning_rate": 1.917241379310345e-05,
+      "loss": 0.9433,
+      "step": 1100
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.9278285503387451,
+      "learning_rate": 1.903448275862069e-05,
+      "loss": 0.9442,
+      "step": 1200
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.8357234001159668,
+      "learning_rate": 1.8896551724137934e-05,
+      "loss": 0.9439,
+      "step": 1300
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.8239058256149292,
+      "learning_rate": 1.8758620689655173e-05,
+      "loss": 0.9428,
+      "step": 1400
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.8225810527801514,
+      "learning_rate": 1.8620689655172415e-05,
+      "loss": 0.945,
+      "step": 1500
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.317204475402832,
+      "learning_rate": 1.8482758620689657e-05,
+      "loss": 0.9333,
+      "step": 1600
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.8159579634666443,
+      "learning_rate": 1.8344827586206896e-05,
+      "loss": 0.9462,
+      "step": 1700
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.8454849123954773,
+      "learning_rate": 1.820689655172414e-05,
+      "loss": 0.9283,
+      "step": 1800
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.8753693103790283,
+      "learning_rate": 1.806896551724138e-05,
+      "loss": 0.9193,
+      "step": 1900
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.808141827583313,
+      "learning_rate": 1.7931034482758623e-05,
+      "loss": 0.9568,
+      "step": 2000
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.9040705561637878,
+      "learning_rate": 1.7793103448275862e-05,
+      "loss": 0.9189,
+      "step": 2100
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.7747160792350769,
+      "learning_rate": 1.7655172413793105e-05,
+      "loss": 0.9285,
+      "step": 2200
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.7610200643539429,
+      "learning_rate": 1.7517241379310347e-05,
+      "loss": 0.9339,
+      "step": 2300
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.8340559601783752,
+      "learning_rate": 1.7379310344827586e-05,
+      "loss": 0.9362,
+      "step": 2400
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.7427037954330444,
+      "learning_rate": 1.7241379310344828e-05,
+      "loss": 0.9302,
+      "step": 2500
+    },
+    {
+      "epoch": 0.5,
+      "eval_loss": 0.9145726561546326,
+      "eval_runtime": 40.1434,
+      "eval_samples_per_second": 24.911,
+      "eval_steps_per_second": 6.228,
+      "step": 2500
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.7336578369140625,
+      "learning_rate": 1.710344827586207e-05,
+      "loss": 0.9184,
+      "step": 2600
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.7962681651115417,
+      "learning_rate": 1.6965517241379313e-05,
+      "loss": 0.9207,
+      "step": 2700
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.7666469812393188,
+      "learning_rate": 1.6827586206896552e-05,
+      "loss": 0.9169,
+      "step": 2800
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.7549653053283691,
+      "learning_rate": 1.6689655172413794e-05,
+      "loss": 0.9231,
+      "step": 2900
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.8372851610183716,
+      "learning_rate": 1.6551724137931037e-05,
+      "loss": 0.9189,
+      "step": 3000
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.9210222363471985,
+      "learning_rate": 1.6413793103448276e-05,
+      "loss": 0.9227,
+      "step": 3100
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.9357787370681763,
+      "learning_rate": 1.6275862068965518e-05,
+      "loss": 0.9192,
+      "step": 3200
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.7520227432250977,
+      "learning_rate": 1.613793103448276e-05,
+      "loss": 0.8994,
+      "step": 3300
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.9676304459571838,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.9105,
+      "step": 3400
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.8729006052017212,
+      "learning_rate": 1.586206896551724e-05,
+      "loss": 0.9184,
+      "step": 3500
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.899910032749176,
+      "learning_rate": 1.5724137931034484e-05,
+      "loss": 0.9051,
+      "step": 3600
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.874957799911499,
+      "learning_rate": 1.5586206896551726e-05,
+      "loss": 0.9252,
+      "step": 3700
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.8180311322212219,
+      "learning_rate": 1.5448275862068965e-05,
+      "loss": 0.9204,
+      "step": 3800
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.8456066846847534,
+      "learning_rate": 1.5310344827586208e-05,
+      "loss": 0.9159,
+      "step": 3900
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.895601212978363,
+      "learning_rate": 1.5172413793103448e-05,
+      "loss": 0.9075,
+      "step": 4000
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.7567282915115356,
+      "learning_rate": 1.503448275862069e-05,
+      "loss": 0.8972,
+      "step": 4100
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.7225831151008606,
+      "learning_rate": 1.4896551724137933e-05,
+      "loss": 0.9193,
+      "step": 4200
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.8456130623817444,
+      "learning_rate": 1.4758620689655174e-05,
+      "loss": 0.9043,
+      "step": 4300
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.7458916306495667,
+      "learning_rate": 1.4620689655172416e-05,
+      "loss": 0.9069,
+      "step": 4400
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.7301814556121826,
+      "learning_rate": 1.4482758620689657e-05,
+      "loss": 0.9143,
+      "step": 4500
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.8358351588249207,
+      "learning_rate": 1.4344827586206897e-05,
+      "loss": 0.9024,
+      "step": 4600
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.9452328085899353,
+      "learning_rate": 1.4206896551724138e-05,
+      "loss": 0.909,
+      "step": 4700
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.7513816952705383,
+      "learning_rate": 1.406896551724138e-05,
+      "loss": 0.9049,
+      "step": 4800
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.7348505258560181,
+      "learning_rate": 1.3931034482758621e-05,
+      "loss": 0.9024,
+      "step": 4900
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.7450273633003235,
+      "learning_rate": 1.3793103448275863e-05,
+      "loss": 0.903,
+      "step": 5000
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.9057763814926147,
+      "eval_runtime": 40.9147,
+      "eval_samples_per_second": 24.441,
+      "eval_steps_per_second": 6.11,
+      "step": 5000
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.8260780572891235,
+      "learning_rate": 1.3655172413793106e-05,
+      "loss": 0.797,
+      "step": 5100
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.9057289958000183,
+      "learning_rate": 1.3517241379310346e-05,
+      "loss": 0.7991,
+      "step": 5200
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.7878429889678955,
+      "learning_rate": 1.3379310344827587e-05,
+      "loss": 0.7986,
+      "step": 5300
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.8928379416465759,
+      "learning_rate": 1.324137931034483e-05,
+      "loss": 0.8137,
+      "step": 5400
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.8249024152755737,
+      "learning_rate": 1.310344827586207e-05,
+      "loss": 0.7987,
+      "step": 5500
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.7873898148536682,
+      "learning_rate": 1.296551724137931e-05,
+      "loss": 0.8021,
+      "step": 5600
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 0.9511479735374451,
+      "learning_rate": 1.2827586206896551e-05,
+      "loss": 0.8004,
+      "step": 5700
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.9020041227340698,
+      "learning_rate": 1.2689655172413795e-05,
+      "loss": 0.7941,
+      "step": 5800
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.8485173583030701,
+      "learning_rate": 1.2551724137931036e-05,
+      "loss": 0.7892,
+      "step": 5900
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.827191174030304,
+      "learning_rate": 1.2413793103448277e-05,
+      "loss": 0.8055,
+      "step": 6000
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.7705573439598083,
+      "learning_rate": 1.2275862068965519e-05,
+      "loss": 0.7992,
+      "step": 6100
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.806735634803772,
+      "learning_rate": 1.213793103448276e-05,
+      "loss": 0.8084,
+      "step": 6200
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.7915472388267517,
+      "learning_rate": 1.2e-05,
+      "loss": 0.7968,
+      "step": 6300
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.8447715044021606,
+      "learning_rate": 1.1862068965517241e-05,
+      "loss": 0.7857,
+      "step": 6400
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.8174277544021606,
+      "learning_rate": 1.1724137931034483e-05,
+      "loss": 0.8081,
+      "step": 6500
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.8464189767837524,
+      "learning_rate": 1.1586206896551726e-05,
+      "loss": 0.7923,
+      "step": 6600
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.8201343417167664,
+      "learning_rate": 1.1448275862068966e-05,
+      "loss": 0.8015,
+      "step": 6700
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 0.834135890007019,
+      "learning_rate": 1.1310344827586209e-05,
+      "loss": 0.7984,
+      "step": 6800
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.8788368701934814,
+      "learning_rate": 1.117241379310345e-05,
+      "loss": 0.7932,
+      "step": 6900
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.7641535401344299,
+      "learning_rate": 1.103448275862069e-05,
+      "loss": 0.7959,
+      "step": 7000
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.8290014863014221,
+      "learning_rate": 1.0896551724137932e-05,
+      "loss": 0.7908,
+      "step": 7100
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.9937741160392761,
+      "learning_rate": 1.0758620689655173e-05,
+      "loss": 0.818,
+      "step": 7200
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 0.8346081972122192,
+      "learning_rate": 1.0620689655172414e-05,
+      "loss": 0.8101,
+      "step": 7300
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.8698582053184509,
+      "learning_rate": 1.0482758620689658e-05,
+      "loss": 0.8128,
+      "step": 7400
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.8040934801101685,
+      "learning_rate": 1.0344827586206898e-05,
+      "loss": 0.7991,
+      "step": 7500
+    },
+    {
+      "epoch": 1.5,
+      "eval_loss": 0.911612331867218,
+      "eval_runtime": 40.9979,
+      "eval_samples_per_second": 24.392,
+      "eval_steps_per_second": 6.098,
+      "step": 7500
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.8177987933158875,
+      "learning_rate": 1.0206896551724139e-05,
+      "loss": 0.7867,
+      "step": 7600
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 0.7633827924728394,
+      "learning_rate": 1.006896551724138e-05,
+      "loss": 0.7916,
+      "step": 7700
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.8105002641677856,
+      "learning_rate": 9.931034482758622e-06,
+      "loss": 0.8111,
+      "step": 7800
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.893203616142273,
+      "learning_rate": 9.793103448275863e-06,
+      "loss": 0.7984,
+      "step": 7900
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.7978829145431519,
+      "learning_rate": 9.655172413793105e-06,
+      "loss": 0.8086,
+      "step": 8000
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.8793106079101562,
+      "learning_rate": 9.517241379310346e-06,
+      "loss": 0.8047,
+      "step": 8100
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 0.8467621803283691,
+      "learning_rate": 9.379310344827586e-06,
+      "loss": 0.8008,
+      "step": 8200
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 0.8126978278160095,
+      "learning_rate": 9.241379310344829e-06,
+      "loss": 0.7989,
+      "step": 8300
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 0.8217782378196716,
+      "learning_rate": 9.10344827586207e-06,
+      "loss": 0.7927,
+      "step": 8400
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.7457430958747864,
+      "learning_rate": 8.965517241379312e-06,
+      "loss": 0.794,
+      "step": 8500
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.8164616823196411,
+      "learning_rate": 8.827586206896552e-06,
+      "loss": 0.8057,
+      "step": 8600
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 0.8319987058639526,
+      "learning_rate": 8.689655172413793e-06,
+      "loss": 0.789,
+      "step": 8700
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.9327766299247742,
+      "learning_rate": 8.553103448275863e-06,
+      "loss": 0.786,
+      "step": 8800
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 0.8273203372955322,
+      "learning_rate": 8.415172413793105e-06,
+      "loss": 0.7883,
+      "step": 8900
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.9450199604034424,
+      "learning_rate": 8.277241379310346e-06,
+      "loss": 0.8003,
+      "step": 9000
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 0.7524954080581665,
+      "learning_rate": 8.139310344827586e-06,
+      "loss": 0.7968,
+      "step": 9100
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 0.8359555602073669,
+      "learning_rate": 8.001379310344829e-06,
+      "loss": 0.801,
+      "step": 9200
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 0.8040117025375366,
+      "learning_rate": 7.86344827586207e-06,
+      "loss": 0.7914,
+      "step": 9300
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.8810254335403442,
+      "learning_rate": 7.725517241379312e-06,
+      "loss": 0.8007,
+      "step": 9400
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.83013516664505,
+      "learning_rate": 7.587586206896552e-06,
+      "loss": 0.7911,
+      "step": 9500
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.8463912010192871,
+      "learning_rate": 7.449655172413793e-06,
+      "loss": 0.793,
+      "step": 9600
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.8582866787910461,
+      "learning_rate": 7.311724137931035e-06,
+      "loss": 0.7897,
+      "step": 9700
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.8311706185340881,
+      "learning_rate": 7.173793103448277e-06,
+      "loss": 0.7946,
+      "step": 9800
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 0.8774682879447937,
+      "learning_rate": 7.0358620689655175e-06,
+      "loss": 0.788,
+      "step": 9900
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.8584392666816711,
+      "learning_rate": 6.897931034482759e-06,
+      "loss": 0.7889,
+      "step": 10000
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.8923286199569702,
+      "eval_runtime": 40.2115,
+      "eval_samples_per_second": 24.869,
+      "eval_steps_per_second": 6.217,
+      "step": 10000
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.8329530954360962,
+      "learning_rate": 6.760000000000001e-06,
+      "loss": 0.7004,
+      "step": 10100
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.8917152881622314,
+      "learning_rate": 6.622068965517242e-06,
+      "loss": 0.695,
+      "step": 10200
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 1.0538383722305298,
+      "learning_rate": 6.4841379310344835e-06,
+      "loss": 0.6897,
+      "step": 10300
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.8790658712387085,
+      "learning_rate": 6.346206896551724e-06,
+      "loss": 0.6994,
+      "step": 10400
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 1.0203146934509277,
+      "learning_rate": 6.209655172413793e-06,
+      "loss": 0.698,
+      "step": 10500
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.8630858063697815,
+      "learning_rate": 6.071724137931035e-06,
+      "loss": 0.6928,
+      "step": 10600
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 0.8968507051467896,
+      "learning_rate": 5.933793103448277e-06,
+      "loss": 0.6933,
+      "step": 10700
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 0.8782464861869812,
+      "learning_rate": 5.7958620689655175e-06,
+      "loss": 0.6946,
+      "step": 10800
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 0.9616362452507019,
+      "learning_rate": 5.657931034482759e-06,
+      "loss": 0.6903,
+      "step": 10900
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.9654956459999084,
+      "learning_rate": 5.5200000000000005e-06,
+      "loss": 0.6919,
+      "step": 11000
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 0.9196248054504395,
+      "learning_rate": 5.382068965517242e-06,
+      "loss": 0.6853,
+      "step": 11100
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.9637214541435242,
+      "learning_rate": 5.2441379310344835e-06,
+      "loss": 0.6927,
+      "step": 11200
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 0.972012460231781,
+      "learning_rate": 5.106206896551724e-06,
+      "loss": 0.697,
+      "step": 11300
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 1.038694143295288,
+      "learning_rate": 4.968275862068966e-06,
+      "loss": 0.6964,
+      "step": 11400
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 0.946384072303772,
+      "learning_rate": 4.830344827586207e-06,
+      "loss": 0.7077,
+      "step": 11500
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 0.897548496723175,
+      "learning_rate": 4.692413793103449e-06,
+      "loss": 0.6941,
+      "step": 11600
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 0.9227349162101746,
+      "learning_rate": 4.55448275862069e-06,
+      "loss": 0.705,
+      "step": 11700
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 0.9196397066116333,
+      "learning_rate": 4.416551724137932e-06,
+      "loss": 0.6989,
+      "step": 11800
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 0.9564076662063599,
+      "learning_rate": 4.278620689655173e-06,
+      "loss": 0.688,
+      "step": 11900
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.9881793260574341,
+      "learning_rate": 4.140689655172414e-06,
+      "loss": 0.6917,
+      "step": 12000
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 0.92747962474823,
+      "learning_rate": 4.002758620689655e-06,
+      "loss": 0.6871,
+      "step": 12100
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.9182614088058472,
+      "learning_rate": 3.864827586206897e-06,
+      "loss": 0.6868,
+      "step": 12200
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 1.0040738582611084,
+      "learning_rate": 3.7268965517241383e-06,
+      "loss": 0.7016,
+      "step": 12300
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.8651072382926941,
+      "learning_rate": 3.5889655172413794e-06,
+      "loss": 0.6906,
+      "step": 12400
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.945471465587616,
+      "learning_rate": 3.4510344827586214e-06,
+      "loss": 0.6963,
+      "step": 12500
+    },
+    {
+      "epoch": 2.5,
+      "eval_loss": 0.8942536115646362,
+      "eval_runtime": 40.9486,
+      "eval_samples_per_second": 24.421,
+      "eval_steps_per_second": 6.105,
+      "step": 12500
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 0.864122748374939,
+      "learning_rate": 3.3131034482758624e-06,
+      "loss": 0.7025,
+      "step": 12600
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 1.153102993965149,
+      "learning_rate": 3.175172413793104e-06,
+      "loss": 0.699,
+      "step": 12700
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.9909245371818542,
+      "learning_rate": 3.037241379310345e-06,
+      "loss": 0.6848,
+      "step": 12800
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 0.9752694368362427,
+      "learning_rate": 2.8993103448275865e-06,
+      "loss": 0.6756,
+      "step": 12900
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 1.0156644582748413,
+      "learning_rate": 2.7613793103448276e-06,
+      "loss": 0.7047,
+      "step": 13000
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.9422524571418762,
+      "learning_rate": 2.6234482758620695e-06,
+      "loss": 0.6999,
+      "step": 13100
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.9309016466140747,
+      "learning_rate": 2.4855172413793106e-06,
+      "loss": 0.6982,
+      "step": 13200
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 0.9932184815406799,
+      "learning_rate": 2.3475862068965517e-06,
+      "loss": 0.6878,
+      "step": 13300
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 1.0034270286560059,
+      "learning_rate": 2.209655172413793e-06,
+      "loss": 0.6943,
+      "step": 13400
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 0.9979943037033081,
+      "learning_rate": 2.0717241379310347e-06,
+      "loss": 0.6874,
+      "step": 13500
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 0.9405415058135986,
+      "learning_rate": 1.933793103448276e-06,
+      "loss": 0.675,
+      "step": 13600
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 0.973407506942749,
+      "learning_rate": 1.7958620689655173e-06,
+      "loss": 0.6751,
+      "step": 13700
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 0.9658547043800354,
+      "learning_rate": 1.6579310344827588e-06,
+      "loss": 0.7019,
+      "step": 13800
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 0.9559837579727173,
+      "learning_rate": 1.52e-06,
+      "loss": 0.6795,
+      "step": 13900
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.9583409428596497,
+      "learning_rate": 1.3820689655172416e-06,
+      "loss": 0.6943,
+      "step": 14000
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 0.8599256873130798,
+      "learning_rate": 1.2441379310344829e-06,
+      "loss": 0.6915,
+      "step": 14100
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 0.9818318486213684,
+      "learning_rate": 1.1062068965517241e-06,
+      "loss": 0.6845,
+      "step": 14200
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 1.0233429670333862,
+      "learning_rate": 9.682758620689656e-07,
+      "loss": 0.6936,
+      "step": 14300
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.9797420501708984,
+      "learning_rate": 8.303448275862069e-07,
+      "loss": 0.6903,
+      "step": 14400
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 0.9917966723442078,
+      "learning_rate": 6.924137931034483e-07,
+      "loss": 0.7028,
+      "step": 14500
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.9207854270935059,
+      "learning_rate": 5.544827586206897e-07,
+      "loss": 0.6739,
+      "step": 14600
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 0.9161053895950317,
+      "learning_rate": 4.1655172413793107e-07,
+      "loss": 0.7003,
+      "step": 14700
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 1.0565011501312256,
+      "learning_rate": 2.7862068965517247e-07,
+      "loss": 0.6872,
+      "step": 14800
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 1.0897576808929443,
+      "learning_rate": 1.406896551724138e-07,
+      "loss": 0.6919,
+      "step": 14900
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.9112550020217896,
+      "learning_rate": 2.758620689655173e-09,
+      "loss": 0.6804,
+      "step": 15000
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.9123731255531311,
+      "eval_runtime": 40.7263,
+      "eval_samples_per_second": 24.554,
+      "eval_steps_per_second": 6.139,
+      "step": 15000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 15000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 2500,
+  "total_flos": 1.88804379967488e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2844010f60d782f48a1430f48583b118c9c59b86c24f41fb14bb3f5174b5022
+size 4920