Upload 10 files

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|end_of_text|>": 32001,
+  "<|im_end|>": 32000,
+  "<|im_start|>": 32002
+}

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "Sicarius-Prototyping/Proto_M7B-4B-RP_demo",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 7168,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": false,
+  "vocab_size": 32003
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.44.0.dev0"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1f1666676a5d42e11739529eaab8d482bb32f44198437a0fbcd3ad2d99798f3
+size 4993657616

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b3765f597551ae77da16fd44101c199034b21b12a2e5011ed10a3684e3c65f3
+size 3852744856

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 8846368768
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,70 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

trainer_state.json

@@ -0,0 +1,503 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9367088607594938,
+  "eval_steps": 8,
+  "global_step": 58,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03375527426160337,
+      "grad_norm": 164.57892531016844,
+      "learning_rate": 1.999348095389677e-05,
+      "loss": 2.5086,
+      "step": 1
+    },
+    {
+      "epoch": 0.03375527426160337,
+      "eval_loss": 61.82543182373047,
+      "eval_runtime": 1145.189,
+      "eval_samples_per_second": 0.339,
+      "eval_steps_per_second": 0.114,
+      "step": 1
+    },
+    {
+      "epoch": 0.06751054852320675,
+      "grad_norm": 2207.8144170196915,
+      "learning_rate": 1.9973932315179502e-05,
+      "loss": 61.7293,
+      "step": 2
+    },
+    {
+      "epoch": 0.10126582278481013,
+      "grad_norm": 756.522802035735,
+      "learning_rate": 1.9941379571543597e-05,
+      "loss": 11.6024,
+      "step": 3
+    },
+    {
+      "epoch": 0.1350210970464135,
+      "grad_norm": 192.54122935101458,
+      "learning_rate": 1.9895865165556375e-05,
+      "loss": 5.4118,
+      "step": 4
+    },
+    {
+      "epoch": 0.16877637130801687,
+      "grad_norm": 79.54768064559613,
+      "learning_rate": 1.9837448439320027e-05,
+      "loss": 4.2774,
+      "step": 5
+    },
+    {
+      "epoch": 0.20253164556962025,
+      "grad_norm": 50.805172249917135,
+      "learning_rate": 1.976620555710087e-05,
+      "loss": 2.9952,
+      "step": 6
+    },
+    {
+      "epoch": 0.23628691983122363,
+      "grad_norm": 190.4579083255668,
+      "learning_rate": 1.9682229406025635e-05,
+      "loss": 3.4715,
+      "step": 7
+    },
+    {
+      "epoch": 0.270042194092827,
+      "grad_norm": 81.40501738510194,
+      "learning_rate": 1.9585629474974413e-05,
+      "loss": 3.0977,
+      "step": 8
+    },
+    {
+      "epoch": 0.270042194092827,
+      "eval_loss": 2.3871593475341797,
+      "eval_runtime": 1143.2965,
+      "eval_samples_per_second": 0.339,
+      "eval_steps_per_second": 0.114,
+      "step": 8
+    },
+    {
+      "epoch": 0.3037974683544304,
+      "grad_norm": 20.30831612560288,
+      "learning_rate": 1.9476531711828027e-05,
+      "loss": 2.3479,
+      "step": 9
+    },
+    {
+      "epoch": 0.33755274261603374,
+      "grad_norm": 69.39654085624514,
+      "learning_rate": 1.935507835925601e-05,
+      "loss": 2.9202,
+      "step": 10
+    },
+    {
+      "epoch": 0.37130801687763715,
+      "grad_norm": 4745.389973437378,
+      "learning_rate": 1.9221427769259333e-05,
+      "loss": 7.0232,
+      "step": 11
+    },
+    {
+      "epoch": 0.4050632911392405,
+      "grad_norm": 21.931966839150423,
+      "learning_rate": 1.9075754196709574e-05,
+      "loss": 2.2551,
+      "step": 12
+    },
+    {
+      "epoch": 0.4388185654008439,
+      "grad_norm": 19.01111638980517,
+      "learning_rate": 1.8918247572153822e-05,
+      "loss": 2.0236,
+      "step": 13
+    },
+    {
+      "epoch": 0.47257383966244726,
+      "grad_norm": 32.39507363625583,
+      "learning_rate": 1.8749113254181498e-05,
+      "loss": 1.8924,
+      "step": 14
+    },
+    {
+      "epoch": 0.5063291139240507,
+      "grad_norm": 16.75690739105817,
+      "learning_rate": 1.8568571761675893e-05,
+      "loss": 1.7989,
+      "step": 15
+    },
+    {
+      "epoch": 0.540084388185654,
+      "grad_norm": 14.061410548076694,
+      "learning_rate": 1.837685848629965e-05,
+      "loss": 1.7912,
+      "step": 16
+    },
+    {
+      "epoch": 0.540084388185654,
+      "eval_loss": 1.723136305809021,
+      "eval_runtime": 1145.6284,
+      "eval_samples_per_second": 0.339,
+      "eval_steps_per_second": 0.113,
+      "step": 16
+    },
+    {
+      "epoch": 0.5738396624472574,
+      "grad_norm": 8.492636688902186,
+      "learning_rate": 1.817422338558892e-05,
+      "loss": 1.6984,
+      "step": 17
+    },
+    {
+      "epoch": 0.6075949367088608,
+      "grad_norm": 25.604058468504437,
+      "learning_rate": 1.796093065705644e-05,
+      "loss": 1.8047,
+      "step": 18
+    },
+    {
+      "epoch": 0.6413502109704642,
+      "grad_norm": 8.054684184801694,
+      "learning_rate": 1.7737258393728363e-05,
+      "loss": 1.6384,
+      "step": 19
+    },
+    {
+      "epoch": 0.6751054852320675,
+      "grad_norm": 11.182600088475377,
+      "learning_rate": 1.7503498221564026e-05,
+      "loss": 1.646,
+      "step": 20
+    },
+    {
+      "epoch": 0.7088607594936709,
+      "grad_norm": 6.767726369227721,
+      "learning_rate": 1.725995491923131e-05,
+      "loss": 1.5879,
+      "step": 21
+    },
+    {
+      "epoch": 0.7426160337552743,
+      "grad_norm": 5.329977390320079,
+      "learning_rate": 1.7006946020733426e-05,
+      "loss": 1.5666,
+      "step": 22
+    },
+    {
+      "epoch": 0.7763713080168776,
+      "grad_norm": 6.221825534043382,
+      "learning_rate": 1.6744801401405138e-05,
+      "loss": 1.5162,
+      "step": 23
+    },
+    {
+      "epoch": 0.810126582278481,
+      "grad_norm": 8.196688788302577,
+      "learning_rate": 1.647386284781828e-05,
+      "loss": 1.4852,
+      "step": 24
+    },
+    {
+      "epoch": 0.810126582278481,
+      "eval_loss": 1.493324875831604,
+      "eval_runtime": 1145.9735,
+      "eval_samples_per_second": 0.339,
+      "eval_steps_per_second": 0.113,
+      "step": 24
+    },
+    {
+      "epoch": 0.8438818565400844,
+      "grad_norm": 4.002901455476497,
+      "learning_rate": 1.6194483612157232e-05,
+      "loss": 1.4674,
+      "step": 25
+    },
+    {
+      "epoch": 0.8776371308016878,
+      "grad_norm": 5.182882804957283,
+      "learning_rate": 1.590702795164551e-05,
+      "loss": 1.4015,
+      "step": 26
+    },
+    {
+      "epoch": 0.9113924050632911,
+      "grad_norm": 4.698321655590912,
+      "learning_rate": 1.5611870653623826e-05,
+      "loss": 1.4439,
+      "step": 27
+    },
+    {
+      "epoch": 0.9451476793248945,
+      "grad_norm": 4.697370183022894,
+      "learning_rate": 1.530939654689887e-05,
+      "loss": 1.4039,
+      "step": 28
+    },
+    {
+      "epoch": 0.9789029535864979,
+      "grad_norm": 5.756016776033281,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 1.4121,
+      "step": 29
+    },
+    {
+      "epoch": 1.0126582278481013,
+      "grad_norm": 4.803347318489069,
+      "learning_rate": 1.4684084406997903e-05,
+      "loss": 1.4221,
+      "step": 30
+    },
+    {
+      "epoch": 1.0253164556962024,
+      "grad_norm": 3.8310946409702917,
+      "learning_rate": 1.4362061661555675e-05,
+      "loss": 1.2795,
+      "step": 31
+    },
+    {
+      "epoch": 1.0590717299578059,
+      "grad_norm": 3.2771871936310886,
+      "learning_rate": 1.4034351619898088e-05,
+      "loss": 1.2648,
+      "step": 32
+    },
+    {
+      "epoch": 1.0590717299578059,
+      "eval_loss": 1.3767977952957153,
+      "eval_runtime": 1142.0223,
+      "eval_samples_per_second": 0.34,
+      "eval_steps_per_second": 0.114,
+      "step": 32
+    },
+    {
+      "epoch": 1.0928270042194093,
+      "grad_norm": 3.6329843131833677,
+      "learning_rate": 1.3701381553399147e-05,
+      "loss": 1.229,
+      "step": 33
+    },
+    {
+      "epoch": 1.1265822784810127,
+      "grad_norm": 2.745855502867358,
+      "learning_rate": 1.3363585591501751e-05,
+      "loss": 1.214,
+      "step": 34
+    },
+    {
+      "epoch": 1.160337552742616,
+      "grad_norm": 2.612920670658094,
+      "learning_rate": 1.3021404155695728e-05,
+      "loss": 1.2036,
+      "step": 35
+    },
+    {
+      "epoch": 1.1940928270042195,
+      "grad_norm": 3.7574311695406277,
+      "learning_rate": 1.2675283385292212e-05,
+      "loss": 1.2269,
+      "step": 36
+    },
+    {
+      "epoch": 1.2278481012658227,
+      "grad_norm": 5.156934842955387,
+      "learning_rate": 1.2325674555743106e-05,
+      "loss": 1.1773,
+      "step": 37
+    },
+    {
+      "epoch": 1.261603375527426,
+      "grad_norm": 2.8644440958111477,
+      "learning_rate": 1.1973033490264e-05,
+      "loss": 1.1767,
+      "step": 38
+    },
+    {
+      "epoch": 1.2953586497890295,
+      "grad_norm": 3.708080726049318,
+      "learning_rate": 1.161781996552765e-05,
+      "loss": 1.1722,
+      "step": 39
+    },
+    {
+      "epoch": 1.3291139240506329,
+      "grad_norm": 5.32705561402287,
+      "learning_rate": 1.1260497112202895e-05,
+      "loss": 1.2019,
+      "step": 40
+    },
+    {
+      "epoch": 1.3291139240506329,
+      "eval_loss": 1.3283240795135498,
+      "eval_runtime": 1141.5031,
+      "eval_samples_per_second": 0.34,
+      "eval_steps_per_second": 0.114,
+      "step": 40
+    },
+    {
+      "epoch": 1.3628691983122363,
+      "grad_norm": 3.2178853179051385,
+      "learning_rate": 1.0901530811120655e-05,
+      "loss": 1.1555,
+      "step": 41
+    },
+    {
+      "epoch": 1.3966244725738397,
+      "grad_norm": 8.771767271309226,
+      "learning_rate": 1.0541389085854177e-05,
+      "loss": 1.1992,
+      "step": 42
+    },
+    {
+      "epoch": 1.4303797468354431,
+      "grad_norm": 4.585556716316959,
+      "learning_rate": 1.0180541492505605e-05,
+      "loss": 1.1806,
+      "step": 43
+    },
+    {
+      "epoch": 1.4641350210970465,
+      "grad_norm": 4.695307422197233,
+      "learning_rate": 9.819458507494395e-06,
+      "loss": 1.1871,
+      "step": 44
+    },
+    {
+      "epoch": 1.49789029535865,
+      "grad_norm": 3.8270032835209427,
+      "learning_rate": 9.458610914145826e-06,
+      "loss": 1.1676,
+      "step": 45
+    },
+    {
+      "epoch": 1.5316455696202531,
+      "grad_norm": 2.9421765052237476,
+      "learning_rate": 9.098469188879348e-06,
+      "loss": 1.1568,
+      "step": 46
+    },
+    {
+      "epoch": 1.5654008438818565,
+      "grad_norm": 3.6707324259149665,
+      "learning_rate": 8.739502887797108e-06,
+      "loss": 1.1623,
+      "step": 47
+    },
+    {
+      "epoch": 1.59915611814346,
+      "grad_norm": 2.512661438115767,
+      "learning_rate": 8.382180034472353e-06,
+      "loss": 1.159,
+      "step": 48
+    },
+    {
+      "epoch": 1.59915611814346,
+      "eval_loss": 1.2873578071594238,
+      "eval_runtime": 1140.5681,
+      "eval_samples_per_second": 0.34,
+      "eval_steps_per_second": 0.114,
+      "step": 48
+    },
+    {
+      "epoch": 1.6329113924050633,
+      "grad_norm": 3.0554382494362677,
+      "learning_rate": 8.026966509736001e-06,
+      "loss": 1.1337,
+      "step": 49
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 2.4338550213067314,
+      "learning_rate": 7.674325444256899e-06,
+      "loss": 1.1403,
+      "step": 50
+    },
+    {
+      "epoch": 1.70042194092827,
+      "grad_norm": 2.5084790446223724,
+      "learning_rate": 7.324716614707794e-06,
+      "loss": 1.1285,
+      "step": 51
+    },
+    {
+      "epoch": 1.7341772151898733,
+      "grad_norm": 2.687074050859683,
+      "learning_rate": 6.978595844304272e-06,
+      "loss": 1.1343,
+      "step": 52
+    },
+    {
+      "epoch": 1.7679324894514767,
+      "grad_norm": 2.3050050678321585,
+      "learning_rate": 6.636414408498249e-06,
+      "loss": 1.1084,
+      "step": 53
+    },
+    {
+      "epoch": 1.8016877637130801,
+      "grad_norm": 2.7567453021542825,
+      "learning_rate": 6.298618446600856e-06,
+      "loss": 1.0955,
+      "step": 54
+    },
+    {
+      "epoch": 1.8354430379746836,
+      "grad_norm": 2.3580908912438443,
+      "learning_rate": 5.965648380101916e-06,
+      "loss": 1.1246,
+      "step": 55
+    },
+    {
+      "epoch": 1.869198312236287,
+      "grad_norm": 2.6030759332802043,
+      "learning_rate": 5.637938338444325e-06,
+      "loss": 1.0977,
+      "step": 56
+    },
+    {
+      "epoch": 1.869198312236287,
+      "eval_loss": 1.2555484771728516,
+      "eval_runtime": 1140.2468,
+      "eval_samples_per_second": 0.34,
+      "eval_steps_per_second": 0.114,
+      "step": 56
+    },
+    {
+      "epoch": 1.9029535864978904,
+      "grad_norm": 2.174215469056319,
+      "learning_rate": 5.3159155930021e-06,
+      "loss": 1.1045,
+      "step": 57
+    },
+    {
+      "epoch": 1.9367088607594938,
+      "grad_norm": 2.2417996699170697,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 1.1341,
+      "step": 58
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 87,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 29,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 31809349877760.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}