Uploaded checkpoint-2500

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-coder-1.3b-instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
+  "rope_theta": 100000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 32256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5bd02ef83525b34694bc97ccd2fb6acedf4a01b1617be4bb645ce9d3fba4f1a
+size 4986380064

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ea8213f434f66992397fcec5ad578f46f862ce1c0c205cc27a1d344f5b7f18
+size 399532808

model.safetensors.index.json

@@ -0,0 +1,226 @@
+{
+  "metadata": {
+    "total_size": 5385887744
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e018c4eb53830317c1f95ca42c944372f3f1c5df5c64829e62629ce4134ca1c1
+size 2699039674

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b07f599d18b490c55cac64640a11117e1d90973d0c87b2688a01864c5778af86
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b02e80c80580d87c5ff39cb04205ae4e434bebaac5d159f2f573cc994135335a
+size 1064

trainer_state.json

@@ -0,0 +1,396 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.0625,
+  "eval_steps": 100,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 11.075758934020996,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 1.4211,
+      "step": 100
+    },
+    {
+      "epoch": 0.0,
+      "eval_loss": 1.2334846258163452,
+      "eval_runtime": 95.5143,
+      "eval_samples_per_second": 10.47,
+      "eval_steps_per_second": 10.47,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 19.510242462158203,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 1.2244,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "eval_loss": 1.1949360370635986,
+      "eval_runtime": 95.601,
+      "eval_samples_per_second": 10.46,
+      "eval_steps_per_second": 10.46,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.514332294464111,
+      "learning_rate": 1.2e-05,
+      "loss": 1.2025,
+      "step": 300
+    },
+    {
+      "epoch": 0.01,
+      "eval_loss": 1.1998025178909302,
+      "eval_runtime": 95.7133,
+      "eval_samples_per_second": 10.448,
+      "eval_steps_per_second": 10.448,
+      "step": 300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 14.404813766479492,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 1.2387,
+      "step": 400
+    },
+    {
+      "epoch": 0.01,
+      "eval_loss": 1.2146040201187134,
+      "eval_runtime": 95.6405,
+      "eval_samples_per_second": 10.456,
+      "eval_steps_per_second": 10.456,
+      "step": 400
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 12.25019645690918,
+      "learning_rate": 2e-05,
+      "loss": 1.2359,
+      "step": 500
+    },
+    {
+      "epoch": 0.01,
+      "eval_loss": 1.232546329498291,
+      "eval_runtime": 95.701,
+      "eval_samples_per_second": 10.449,
+      "eval_steps_per_second": 10.449,
+      "step": 500
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 5.677526473999023,
+      "learning_rate": 1.98974358974359e-05,
+      "loss": 1.2189,
+      "step": 600
+    },
+    {
+      "epoch": 0.01,
+      "eval_loss": 1.2301095724105835,
+      "eval_runtime": 93.7117,
+      "eval_samples_per_second": 10.671,
+      "eval_steps_per_second": 10.671,
+      "step": 600
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 13.75782585144043,
+      "learning_rate": 1.9794871794871798e-05,
+      "loss": 1.1982,
+      "step": 700
+    },
+    {
+      "epoch": 0.02,
+      "eval_loss": 1.2097805738449097,
+      "eval_runtime": 95.0989,
+      "eval_samples_per_second": 10.515,
+      "eval_steps_per_second": 10.515,
+      "step": 700
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.497257232666016,
+      "learning_rate": 1.9692307692307696e-05,
+      "loss": 1.178,
+      "step": 800
+    },
+    {
+      "epoch": 0.02,
+      "eval_loss": 1.1660981178283691,
+      "eval_runtime": 95.0124,
+      "eval_samples_per_second": 10.525,
+      "eval_steps_per_second": 10.525,
+      "step": 800
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.337695121765137,
+      "learning_rate": 1.958974358974359e-05,
+      "loss": 1.1852,
+      "step": 900
+    },
+    {
+      "epoch": 0.02,
+      "eval_loss": 1.1449871063232422,
+      "eval_runtime": 94.9978,
+      "eval_samples_per_second": 10.527,
+      "eval_steps_per_second": 10.527,
+      "step": 900
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 4.1737895011901855,
+      "learning_rate": 1.9487179487179488e-05,
+      "loss": 1.1056,
+      "step": 1000
+    },
+    {
+      "epoch": 0.03,
+      "eval_loss": 1.1298564672470093,
+      "eval_runtime": 94.9792,
+      "eval_samples_per_second": 10.529,
+      "eval_steps_per_second": 10.529,
+      "step": 1000
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 6.821949481964111,
+      "learning_rate": 1.9384615384615386e-05,
+      "loss": 1.1681,
+      "step": 1100
+    },
+    {
+      "epoch": 0.03,
+      "eval_loss": 1.1448038816452026,
+      "eval_runtime": 94.8638,
+      "eval_samples_per_second": 10.541,
+      "eval_steps_per_second": 10.541,
+      "step": 1100
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 3.908639907836914,
+      "learning_rate": 1.9282051282051284e-05,
+      "loss": 1.1311,
+      "step": 1200
+    },
+    {
+      "epoch": 0.03,
+      "eval_loss": 1.0957874059677124,
+      "eval_runtime": 94.6454,
+      "eval_samples_per_second": 10.566,
+      "eval_steps_per_second": 10.566,
+      "step": 1200
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 5.896665096282959,
+      "learning_rate": 1.9179487179487182e-05,
+      "loss": 1.134,
+      "step": 1300
+    },
+    {
+      "epoch": 0.03,
+      "eval_loss": 1.0930886268615723,
+      "eval_runtime": 94.6326,
+      "eval_samples_per_second": 10.567,
+      "eval_steps_per_second": 10.567,
+      "step": 1300
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 5.936835765838623,
+      "learning_rate": 1.907692307692308e-05,
+      "loss": 1.0822,
+      "step": 1400
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 1.1114310026168823,
+      "eval_runtime": 94.6679,
+      "eval_samples_per_second": 10.563,
+      "eval_steps_per_second": 10.563,
+      "step": 1400
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.07722806930542,
+      "learning_rate": 1.8974358974358975e-05,
+      "loss": 1.0924,
+      "step": 1500
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 1.1060781478881836,
+      "eval_runtime": 94.6419,
+      "eval_samples_per_second": 10.566,
+      "eval_steps_per_second": 10.566,
+      "step": 1500
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 8.239760398864746,
+      "learning_rate": 1.8871794871794873e-05,
+      "loss": 1.0801,
+      "step": 1600
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 1.0673882961273193,
+      "eval_runtime": 94.6026,
+      "eval_samples_per_second": 10.571,
+      "eval_steps_per_second": 10.571,
+      "step": 1600
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.63267707824707,
+      "learning_rate": 1.876923076923077e-05,
+      "loss": 1.1051,
+      "step": 1700
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 1.0620085000991821,
+      "eval_runtime": 94.6137,
+      "eval_samples_per_second": 10.569,
+      "eval_steps_per_second": 10.569,
+      "step": 1700
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 5.941944122314453,
+      "learning_rate": 1.866666666666667e-05,
+      "loss": 1.081,
+      "step": 1800
+    },
+    {
+      "epoch": 0.04,
+      "eval_loss": 1.0232795476913452,
+      "eval_runtime": 94.6201,
+      "eval_samples_per_second": 10.569,
+      "eval_steps_per_second": 10.569,
+      "step": 1800
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 4.456423282623291,
+      "learning_rate": 1.8564102564102567e-05,
+      "loss": 1.0303,
+      "step": 1900
+    },
+    {
+      "epoch": 0.05,
+      "eval_loss": 1.052198052406311,
+      "eval_runtime": 94.6135,
+      "eval_samples_per_second": 10.569,
+      "eval_steps_per_second": 10.569,
+      "step": 1900
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 8.572375297546387,
+      "learning_rate": 1.8461538461538465e-05,
+      "loss": 1.048,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05,
+      "eval_loss": 1.0538784265518188,
+      "eval_runtime": 94.6091,
+      "eval_samples_per_second": 10.57,
+      "eval_steps_per_second": 10.57,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 4.592229843139648,
+      "learning_rate": 1.835897435897436e-05,
+      "loss": 1.0623,
+      "step": 2100
+    },
+    {
+      "epoch": 0.05,
+      "eval_loss": 1.0412269830703735,
+      "eval_runtime": 94.6145,
+      "eval_samples_per_second": 10.569,
+      "eval_steps_per_second": 10.569,
+      "step": 2100
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 3.802849054336548,
+      "learning_rate": 1.8256410256410257e-05,
+      "loss": 1.0699,
+      "step": 2200
+    },
+    {
+      "epoch": 0.06,
+      "eval_loss": 1.0224683284759521,
+      "eval_runtime": 94.6257,
+      "eval_samples_per_second": 10.568,
+      "eval_steps_per_second": 10.568,
+      "step": 2200
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 4.145386695861816,
+      "learning_rate": 1.8153846153846155e-05,
+      "loss": 0.9979,
+      "step": 2300
+    },
+    {
+      "epoch": 0.06,
+      "eval_loss": 1.0313537120819092,
+      "eval_runtime": 94.6038,
+      "eval_samples_per_second": 10.57,
+      "eval_steps_per_second": 10.57,
+      "step": 2300
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 4.273189067840576,
+      "learning_rate": 1.8051282051282053e-05,
+      "loss": 1.052,
+      "step": 2400
+    },
+    {
+      "epoch": 0.06,
+      "eval_loss": 1.0333129167556763,
+      "eval_runtime": 94.5812,
+      "eval_samples_per_second": 10.573,
+      "eval_steps_per_second": 10.573,
+      "step": 2400
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 6.093295574188232,
+      "learning_rate": 1.794871794871795e-05,
+      "loss": 1.0025,
+      "step": 2500
+    },
+    {
+      "epoch": 0.06,
+      "eval_loss": 0.9930676817893982,
+      "eval_runtime": 94.5844,
+      "eval_samples_per_second": 10.573,
+      "eval_steps_per_second": 10.573,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 20000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 2500,
+  "total_flos": 3.933424582656e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58f96cee7b3928934c63b8523b5cd7bdf09666d503f89f715c81697ecd9b70dd
+size 5048