Training in progress, step 58, checkpoint

checkpoint-58/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endofprompt|>": 100276,
+  "<|im_end|>": 100265,
+  "<|im_start|>": 100264
+}

checkpoint-58/chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

checkpoint-58/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "eos_token_id": 100257,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 262144,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 128000000,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.1",
+  "use_cache": false,
+  "vocab_size": 102400
+}

checkpoint-58/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.53.1"
+}

checkpoint-58/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8526f3d2dda7ababe38a38c547816f7e1a7055508a227f2129fecfa75bdc678f
+size 4983077832

checkpoint-58/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bab9e2a4d8e7ee07094fe1f1488483e5018d57fda2196ed456ed1cb8ec36a51
+size 3842234168

checkpoint-58/model.safetensors.index.json

@@ -0,0 +1,262 @@
+{
+  "metadata": {
+    "total_parameters": 4412641280,
+    "total_size": 8825282560
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-58/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6efe586e29b7c66e120b95be0e2eb019f1e5294183d44d1b15167f3efcfbbe5
+size 11466962878

checkpoint-58/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b6047bd8cc6f4cdf7c46dea47edb8e542435510070c6cd1e0a7d9ccf5fd7da
+size 14244

checkpoint-58/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c8d9d53ba29a68850b5266b1aaf9ce5b1a0946411953c68dfda72d66a6e964
+size 1064

checkpoint-58/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-58/tokenizer_config.json

@@ -0,0 +1,69 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-58/trainer_state.json

@@ -0,0 +1,440 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 58,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017241379310344827,
+      "grad_norm": 1.34375,
+      "learning_rate": 0.0,
+      "loss": 2.5553,
+      "step": 1
+    },
+    {
+      "epoch": 0.034482758620689655,
+      "grad_norm": 1.359375,
+      "learning_rate": 1.25e-06,
+      "loss": 2.5516,
+      "step": 2
+    },
+    {
+      "epoch": 0.05172413793103448,
+      "grad_norm": 1.25,
+      "learning_rate": 2.5e-06,
+      "loss": 2.5156,
+      "step": 3
+    },
+    {
+      "epoch": 0.06896551724137931,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 2.5199,
+      "step": 4
+    },
+    {
+      "epoch": 0.08620689655172414,
+      "grad_norm": 1.1640625,
+      "learning_rate": 5e-06,
+      "loss": 2.5059,
+      "step": 5
+    },
+    {
+      "epoch": 0.10344827586206896,
+      "grad_norm": 1.2109375,
+      "learning_rate": 6.25e-06,
+      "loss": 2.4909,
+      "step": 6
+    },
+    {
+      "epoch": 0.1206896551724138,
+      "grad_norm": 1.03125,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 2.5399,
+      "step": 7
+    },
+    {
+      "epoch": 0.13793103448275862,
+      "grad_norm": 0.91796875,
+      "learning_rate": 8.750000000000001e-06,
+      "loss": 2.5355,
+      "step": 8
+    },
+    {
+      "epoch": 0.15517241379310345,
+      "grad_norm": 0.828125,
+      "learning_rate": 1e-05,
+      "loss": 2.5359,
+      "step": 9
+    },
+    {
+      "epoch": 0.1724137931034483,
+      "grad_norm": 0.70703125,
+      "learning_rate": 9.99910461334869e-06,
+      "loss": 2.5257,
+      "step": 10
+    },
+    {
+      "epoch": 0.1896551724137931,
+      "grad_norm": 0.6875,
+      "learning_rate": 9.996418774081658e-06,
+      "loss": 2.5138,
+      "step": 11
+    },
+    {
+      "epoch": 0.20689655172413793,
+      "grad_norm": 0.6953125,
+      "learning_rate": 9.991943444144758e-06,
+      "loss": 2.5139,
+      "step": 12
+    },
+    {
+      "epoch": 0.22413793103448276,
+      "grad_norm": 0.87109375,
+      "learning_rate": 9.985680226398261e-06,
+      "loss": 2.4983,
+      "step": 13
+    },
+    {
+      "epoch": 0.2413793103448276,
+      "grad_norm": 0.69921875,
+      "learning_rate": 9.977631364042796e-06,
+      "loss": 2.5036,
+      "step": 14
+    },
+    {
+      "epoch": 0.25862068965517243,
+      "grad_norm": 0.703125,
+      "learning_rate": 9.967799739815925e-06,
+      "loss": 2.5121,
+      "step": 15
+    },
+    {
+      "epoch": 0.27586206896551724,
+      "grad_norm": 0.703125,
+      "learning_rate": 9.956188874959686e-06,
+      "loss": 2.539,
+      "step": 16
+    },
+    {
+      "epoch": 0.29310344827586204,
+      "grad_norm": 0.70703125,
+      "learning_rate": 9.942802927959444e-06,
+      "loss": 2.5107,
+      "step": 17
+    },
+    {
+      "epoch": 0.3103448275862069,
+      "grad_norm": 0.67578125,
+      "learning_rate": 9.927646693054498e-06,
+      "loss": 2.4907,
+      "step": 18
+    },
+    {
+      "epoch": 0.3275862068965517,
+      "grad_norm": 0.66015625,
+      "learning_rate": 9.910725598521014e-06,
+      "loss": 2.4836,
+      "step": 19
+    },
+    {
+      "epoch": 0.3448275862068966,
+      "grad_norm": 0.66796875,
+      "learning_rate": 9.892045704727864e-06,
+      "loss": 2.494,
+      "step": 20
+    },
+    {
+      "epoch": 0.3620689655172414,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.871613701966067e-06,
+      "loss": 2.4733,
+      "step": 21
+    },
+    {
+      "epoch": 0.3793103448275862,
+      "grad_norm": 0.66796875,
+      "learning_rate": 9.849436908052636e-06,
+      "loss": 2.4141,
+      "step": 22
+    },
+    {
+      "epoch": 0.39655172413793105,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.825523265709667e-06,
+      "loss": 2.521,
+      "step": 23
+    },
+    {
+      "epoch": 0.41379310344827586,
+      "grad_norm": 0.640625,
+      "learning_rate": 9.799881339719615e-06,
+      "loss": 2.4647,
+      "step": 24
+    },
+    {
+      "epoch": 0.43103448275862066,
+      "grad_norm": 0.62109375,
+      "learning_rate": 9.772520313857777e-06,
+      "loss": 2.4892,
+      "step": 25
+    },
+    {
+      "epoch": 0.4482758620689655,
+      "grad_norm": 0.6015625,
+      "learning_rate": 9.743449987603082e-06,
+      "loss": 2.4734,
+      "step": 26
+    },
+    {
+      "epoch": 0.46551724137931033,
+      "grad_norm": 0.609375,
+      "learning_rate": 9.712680772628365e-06,
+      "loss": 2.5229,
+      "step": 27
+    },
+    {
+      "epoch": 0.4827586206896552,
+      "grad_norm": 0.6171875,
+      "learning_rate": 9.680223689071364e-06,
+      "loss": 2.5502,
+      "step": 28
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.6171875,
+      "learning_rate": 9.646090361587828e-06,
+      "loss": 2.5008,
+      "step": 29
+    },
+    {
+      "epoch": 0.5172413793103449,
+      "grad_norm": 0.63671875,
+      "learning_rate": 9.610293015188067e-06,
+      "loss": 2.5319,
+      "step": 30
+    },
+    {
+      "epoch": 0.5344827586206896,
+      "grad_norm": 0.61328125,
+      "learning_rate": 9.572844470858537e-06,
+      "loss": 2.5004,
+      "step": 31
+    },
+    {
+      "epoch": 0.5517241379310345,
+      "grad_norm": 0.6171875,
+      "learning_rate": 9.533758140969913e-06,
+      "loss": 2.4611,
+      "step": 32
+    },
+    {
+      "epoch": 0.5689655172413793,
+      "grad_norm": 0.625,
+      "learning_rate": 9.493048024473413e-06,
+      "loss": 2.4883,
+      "step": 33
+    },
+    {
+      "epoch": 0.5862068965517241,
+      "grad_norm": 0.59765625,
+      "learning_rate": 9.450728701886985e-06,
+      "loss": 2.5014,
+      "step": 34
+    },
+    {
+      "epoch": 0.603448275862069,
+      "grad_norm": 0.59765625,
+      "learning_rate": 9.406815330073244e-06,
+      "loss": 2.4959,
+      "step": 35
+    },
+    {
+      "epoch": 0.6206896551724138,
+      "grad_norm": 0.60546875,
+      "learning_rate": 9.36132363681097e-06,
+      "loss": 2.4901,
+      "step": 36
+    },
+    {
+      "epoch": 0.6379310344827587,
+      "grad_norm": 0.59765625,
+      "learning_rate": 9.314269915162115e-06,
+      "loss": 2.54,
+      "step": 37
+    },
+    {
+      "epoch": 0.6551724137931034,
+      "grad_norm": 0.609375,
+      "learning_rate": 9.265671017636384e-06,
+      "loss": 2.5077,
+      "step": 38
+    },
+    {
+      "epoch": 0.6724137931034483,
+      "grad_norm": 0.60546875,
+      "learning_rate": 9.215544350155423e-06,
+      "loss": 2.5093,
+      "step": 39
+    },
+    {
+      "epoch": 0.6896551724137931,
+      "grad_norm": 0.6015625,
+      "learning_rate": 9.163907865818806e-06,
+      "loss": 2.5134,
+      "step": 40
+    },
+    {
+      "epoch": 0.7068965517241379,
+      "grad_norm": 0.609375,
+      "learning_rate": 9.110780058474052e-06,
+      "loss": 2.5651,
+      "step": 41
+    },
+    {
+      "epoch": 0.7241379310344828,
+      "grad_norm": 0.59375,
+      "learning_rate": 9.056179956092961e-06,
+      "loss": 2.4615,
+      "step": 42
+    },
+    {
+      "epoch": 0.7413793103448276,
+      "grad_norm": 0.59375,
+      "learning_rate": 9.000127113956673e-06,
+      "loss": 2.4763,
+      "step": 43
+    },
+    {
+      "epoch": 0.7586206896551724,
+      "grad_norm": 0.58203125,
+      "learning_rate": 8.94264160765183e-06,
+      "loss": 2.4732,
+      "step": 44
+    },
+    {
+      "epoch": 0.7758620689655172,
+      "grad_norm": 0.58203125,
+      "learning_rate": 8.883744025880429e-06,
+      "loss": 2.49,
+      "step": 45
+    },
+    {
+      "epoch": 0.7931034482758621,
+      "grad_norm": 0.5859375,
+      "learning_rate": 8.823455463085873e-06,
+      "loss": 2.4944,
+      "step": 46
+    },
+    {
+      "epoch": 0.8103448275862069,
+      "grad_norm": 0.58203125,
+      "learning_rate": 8.761797511897907e-06,
+      "loss": 2.5177,
+      "step": 47
+    },
+    {
+      "epoch": 0.8275862068965517,
+      "grad_norm": 0.6015625,
+      "learning_rate": 8.698792255399104e-06,
+      "loss": 2.5083,
+      "step": 48
+    },
+    {
+      "epoch": 0.8448275862068966,
+      "grad_norm": 0.58984375,
+      "learning_rate": 8.634462259215719e-06,
+      "loss": 2.5096,
+      "step": 49
+    },
+    {
+      "epoch": 0.8620689655172413,
+      "grad_norm": 0.60546875,
+      "learning_rate": 8.568830563435695e-06,
+      "loss": 2.5267,
+      "step": 50
+    },
+    {
+      "epoch": 0.8793103448275862,
+      "grad_norm": 0.6015625,
+      "learning_rate": 8.501920674356755e-06,
+      "loss": 2.5041,
+      "step": 51
+    },
+    {
+      "epoch": 0.896551724137931,
+      "grad_norm": 0.59765625,
+      "learning_rate": 8.433756556067506e-06,
+      "loss": 2.5081,
+      "step": 52
+    },
+    {
+      "epoch": 0.9137931034482759,
+      "grad_norm": 0.578125,
+      "learning_rate": 8.364362621864595e-06,
+      "loss": 2.4329,
+      "step": 53
+    },
+    {
+      "epoch": 0.9310344827586207,
+      "grad_norm": 0.58984375,
+      "learning_rate": 8.29376372550897e-06,
+      "loss": 2.4784,
+      "step": 54
+    },
+    {
+      "epoch": 0.9482758620689655,
+      "grad_norm": 0.60546875,
+      "learning_rate": 8.221985152324385e-06,
+      "loss": 2.5528,
+      "step": 55
+    },
+    {
+      "epoch": 0.9655172413793104,
+      "grad_norm": 0.59375,
+      "learning_rate": 8.149052610141357e-06,
+      "loss": 2.4803,
+      "step": 56
+    },
+    {
+      "epoch": 0.9827586206896551,
+      "grad_norm": 0.578125,
+      "learning_rate": 8.07499222008977e-06,
+      "loss": 2.5519,
+      "step": 57
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 7.625,
+      "learning_rate": 7.999830507243478e-06,
+      "loss": 3.0347,
+      "step": 58
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 174,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 58,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.587894225456333e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-58/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0be3f349f7a3fd9e30466668e6812794f83e6bab370c1a1e1650c7f1f413982c
+size 6712