update to 5K steps

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "/home/slseanwu/11891_codegen/air2/runs/deepseek-1.3b-full-ghcode-PT-promptC12-bs32-30K-240416/checkpoint-15000",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
+  "rope_theta": 100000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 32256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d497b969803103153cb6b272f727eacea7e628b24bf90bcbbd4b0c53b1fa042c
+size 4986380064

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55a4d0a52b74505d3956fbd85f50cd3be8a1bb0f2f1a1feae5570559b6709a9c
+size 399532808

model.safetensors.index.json

@@ -0,0 +1,226 @@
+{
+  "metadata": {
+    "total_size": 5385887744
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52fe790609a91186e6c87272b023b217e06e90f21a83280a173056a4c80d646c
+size 2699039674

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fddbe326fadb3ce8c449c2642e8a1dd894d5eabee5fd4473ac2c38deccaf19b
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2be67f6aac8e482bb2022409709d8774ffb125292c0c9cf025c0ae747f3a6d57
+size 1064

trainer_state.json

@@ -0,0 +1,451 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 8.605851979345955,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.17,
+      "grad_norm": 1.4102528095245361,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.49,
+      "step": 100
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 1.4677144289016724,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.4121,
+      "step": 200
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.1543195247650146,
+      "learning_rate": 1.2e-05,
+      "loss": 0.3923,
+      "step": 300
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 1.4993153810501099,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.3778,
+      "step": 400
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 1.2486151456832886,
+      "learning_rate": 2e-05,
+      "loss": 0.3529,
+      "step": 500
+    },
+    {
+      "epoch": 0.86,
+      "eval_loss": 0.2814500629901886,
+      "eval_runtime": 65.1492,
+      "eval_samples_per_second": 15.349,
+      "eval_steps_per_second": 3.837,
+      "step": 500
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 1.2084145545959473,
+      "learning_rate": 1.9555555555555557e-05,
+      "loss": 0.3377,
+      "step": 600
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.07809317111969,
+      "learning_rate": 1.9111111111111113e-05,
+      "loss": 0.2781,
+      "step": 700
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 1.1827564239501953,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 0.2763,
+      "step": 800
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 1.1633341312408447,
+      "learning_rate": 1.8222222222222224e-05,
+      "loss": 0.2889,
+      "step": 900
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 1.2006715536117554,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 0.2642,
+      "step": 1000
+    },
+    {
+      "epoch": 1.72,
+      "eval_loss": 0.2325076013803482,
+      "eval_runtime": 65.1357,
+      "eval_samples_per_second": 15.353,
+      "eval_steps_per_second": 3.838,
+      "step": 1000
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 1.0107322931289673,
+      "learning_rate": 1.7333333333333336e-05,
+      "loss": 0.2561,
+      "step": 1100
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 1.0967727899551392,
+      "learning_rate": 1.688888888888889e-05,
+      "loss": 0.2192,
+      "step": 1200
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 1.5093573331832886,
+      "learning_rate": 1.6444444444444444e-05,
+      "loss": 0.1575,
+      "step": 1300
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 1.5922331809997559,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.1534,
+      "step": 1400
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 1.45039963722229,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 0.1521,
+      "step": 1500
+    },
+    {
+      "epoch": 2.58,
+      "eval_loss": 0.23398388922214508,
+      "eval_runtime": 65.1855,
+      "eval_samples_per_second": 15.341,
+      "eval_steps_per_second": 3.835,
+      "step": 1500
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.214897871017456,
+      "learning_rate": 1.5111111111111112e-05,
+      "loss": 0.1603,
+      "step": 1600
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 1.1699700355529785,
+      "learning_rate": 1.4666666666666666e-05,
+      "loss": 0.1613,
+      "step": 1700
+    },
+    {
+      "epoch": 3.1,
+      "grad_norm": 1.2094334363937378,
+      "learning_rate": 1.4222222222222224e-05,
+      "loss": 0.1101,
+      "step": 1800
+    },
+    {
+      "epoch": 3.27,
+      "grad_norm": 1.064051866531372,
+      "learning_rate": 1.377777777777778e-05,
+      "loss": 0.082,
+      "step": 1900
+    },
+    {
+      "epoch": 3.44,
+      "grad_norm": 1.6143615245819092,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.0836,
+      "step": 2000
+    },
+    {
+      "epoch": 3.44,
+      "eval_loss": 0.2621379792690277,
+      "eval_runtime": 65.097,
+      "eval_samples_per_second": 15.362,
+      "eval_steps_per_second": 3.84,
+      "step": 2000
+    },
+    {
+      "epoch": 3.61,
+      "grad_norm": 1.1602758169174194,
+      "learning_rate": 1.288888888888889e-05,
+      "loss": 0.0851,
+      "step": 2100
+    },
+    {
+      "epoch": 3.79,
+      "grad_norm": 1.2806442975997925,
+      "learning_rate": 1.2444444444444446e-05,
+      "loss": 0.081,
+      "step": 2200
+    },
+    {
+      "epoch": 3.96,
+      "grad_norm": 1.0125651359558105,
+      "learning_rate": 1.2e-05,
+      "loss": 0.0809,
+      "step": 2300
+    },
+    {
+      "epoch": 4.13,
+      "grad_norm": 1.0691075325012207,
+      "learning_rate": 1.1555555555555556e-05,
+      "loss": 0.0507,
+      "step": 2400
+    },
+    {
+      "epoch": 4.3,
+      "grad_norm": 1.205142617225647,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 0.041,
+      "step": 2500
+    },
+    {
+      "epoch": 4.3,
+      "eval_loss": 0.3192918598651886,
+      "eval_runtime": 65.1967,
+      "eval_samples_per_second": 15.338,
+      "eval_steps_per_second": 3.835,
+      "step": 2500
+    },
+    {
+      "epoch": 4.48,
+      "grad_norm": 1.4642832279205322,
+      "learning_rate": 1.0666666666666667e-05,
+      "loss": 0.041,
+      "step": 2600
+    },
+    {
+      "epoch": 4.65,
+      "grad_norm": 0.89285808801651,
+      "learning_rate": 1.0222222222222223e-05,
+      "loss": 0.0417,
+      "step": 2700
+    },
+    {
+      "epoch": 4.82,
+      "grad_norm": 1.5299988985061646,
+      "learning_rate": 9.777777777777779e-06,
+      "loss": 0.0437,
+      "step": 2800
+    },
+    {
+      "epoch": 4.99,
+      "grad_norm": 1.0683481693267822,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 0.0424,
+      "step": 2900
+    },
+    {
+      "epoch": 5.16,
+      "grad_norm": 0.7894615530967712,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 0.0193,
+      "step": 3000
+    },
+    {
+      "epoch": 5.16,
+      "eval_loss": 0.37849658727645874,
+      "eval_runtime": 65.1259,
+      "eval_samples_per_second": 15.355,
+      "eval_steps_per_second": 3.839,
+      "step": 3000
+    },
+    {
+      "epoch": 5.34,
+      "grad_norm": 1.2226183414459229,
+      "learning_rate": 8.444444444444446e-06,
+      "loss": 0.0201,
+      "step": 3100
+    },
+    {
+      "epoch": 5.51,
+      "grad_norm": 0.9628532528877258,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.0209,
+      "step": 3200
+    },
+    {
+      "epoch": 5.68,
+      "grad_norm": 0.532409131526947,
+      "learning_rate": 7.555555555555556e-06,
+      "loss": 0.0204,
+      "step": 3300
+    },
+    {
+      "epoch": 5.85,
+      "grad_norm": 0.9006184339523315,
+      "learning_rate": 7.111111111111112e-06,
+      "loss": 0.0218,
+      "step": 3400
+    },
+    {
+      "epoch": 6.02,
+      "grad_norm": 0.44430091977119446,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.0197,
+      "step": 3500
+    },
+    {
+      "epoch": 6.02,
+      "eval_loss": 0.35589635372161865,
+      "eval_runtime": 65.2107,
+      "eval_samples_per_second": 15.335,
+      "eval_steps_per_second": 3.834,
+      "step": 3500
+    },
+    {
+      "epoch": 6.2,
+      "grad_norm": 0.3492151200771332,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 0.0083,
+      "step": 3600
+    },
+    {
+      "epoch": 6.37,
+      "grad_norm": 0.5491042137145996,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 0.0098,
+      "step": 3700
+    },
+    {
+      "epoch": 6.54,
+      "grad_norm": 0.6669167876243591,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 0.0087,
+      "step": 3800
+    },
+    {
+      "epoch": 6.71,
+      "grad_norm": 0.9900159239768982,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.0136,
+      "step": 3900
+    },
+    {
+      "epoch": 6.88,
+      "grad_norm": 0.5047167539596558,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.0091,
+      "step": 4000
+    },
+    {
+      "epoch": 6.88,
+      "eval_loss": 0.3709670305252075,
+      "eval_runtime": 65.4136,
+      "eval_samples_per_second": 15.287,
+      "eval_steps_per_second": 3.822,
+      "step": 4000
+    },
+    {
+      "epoch": 7.06,
+      "grad_norm": 0.44396349787712097,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0074,
+      "step": 4100
+    },
+    {
+      "epoch": 7.23,
+      "grad_norm": 0.24100346863269806,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 0.0049,
+      "step": 4200
+    },
+    {
+      "epoch": 7.4,
+      "grad_norm": 0.1909327656030655,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 0.0039,
+      "step": 4300
+    },
+    {
+      "epoch": 7.57,
+      "grad_norm": 0.26105764508247375,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0036,
+      "step": 4400
+    },
+    {
+      "epoch": 7.75,
+      "grad_norm": 0.5079989433288574,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 0.0036,
+      "step": 4500
+    },
+    {
+      "epoch": 7.75,
+      "eval_loss": 0.3895440101623535,
+      "eval_runtime": 65.2965,
+      "eval_samples_per_second": 15.315,
+      "eval_steps_per_second": 3.829,
+      "step": 4500
+    },
+    {
+      "epoch": 7.92,
+      "grad_norm": 0.1813237965106964,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 0.0043,
+      "step": 4600
+    },
+    {
+      "epoch": 8.09,
+      "grad_norm": 0.07837748527526855,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0043,
+      "step": 4700
+    },
+    {
+      "epoch": 8.26,
+      "grad_norm": 0.20979875326156616,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 0.0031,
+      "step": 4800
+    },
+    {
+      "epoch": 8.43,
+      "grad_norm": 0.11709707230329514,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 0.0019,
+      "step": 4900
+    },
+    {
+      "epoch": 8.61,
+      "grad_norm": 0.14616499841213226,
+      "learning_rate": 0.0,
+      "loss": 0.002,
+      "step": 5000
+    },
+    {
+      "epoch": 8.61,
+      "eval_loss": 0.4330046772956848,
+      "eval_runtime": 65.3173,
+      "eval_samples_per_second": 15.31,
+      "eval_steps_per_second": 3.827,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 5000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9,
+  "save_steps": 500,
+  "total_flos": 1.258569996863275e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5df2ea58ab1f1bbfb3de115691887f9de8900b20b0877d4316fdb3b7c2a165cc
+size 4984