Upload GPTNeoXForCausalLM

config.json

@@ -10,21 +10,21 @@
   "eos_token_id": 0,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
-  "hidden_size": 2048,
+  "hidden_size": 8192,
   "initializer_range": 0.02,
-  "intermediate_size": 7168,
+  "intermediate_size": 57344,
   "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 2048,
+  "max_position_embeddings": 8192,
   "model_type": "gpt_neox",
   "num_attention_heads": 8,
   "num_hidden_layers": 8,
   "pad_token_id": 0,
   "rope_scaling": null,
   "rotary_emb_base": 10000,
-  "rotary_pct": 0.75,
+  "rotary_pct": 0.5,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.41.2",
   "use_cache": true,
   "use_parallel_residual": true,
   "vocab_size": 50281

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 0,
   "eos_token_id": 0,
   "pad_token_id": 0,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.41.2"
 }

model-00001-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bad7d323e022be67947ab92b8078959d0d19f764ada75765dcb75eda01534b3
+size 4600890664

model-00002-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e66bb3c5a5af5c43297c7d422de5a9b5c0a956db16cbf9d53c14213d4cc54369
+size 4832363936

model-00003-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c2d1d3ba1eaaf2a145a577ed2f721ea67a79a55e46cde272656573baa35ab66
+size 4832363936

model-00004-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5752b66df6bfbc36dc0b5a26a923da04e325f83505f361946ab8a654cae8f573
+size 4832363936

model-00005-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cf69db59e3ee531b98a1c82df7088a2719267a478c8b2e92f96d77dd09df41c
+size 4832363936

model-00006-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd9501de401cec0f7689c91be965089cd678684d2357f1fb44ee70223f467bb6
+size 4832363936

model-00007-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b138c0b41d39072082401cacbaa5ebed9442528f3ff114ce346eec9263bcc5d0
+size 4832363936

model-00008-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3ed780c195df889b48327855719c2dc158f3ddab539696328db9e46671b170c
+size 4832363936

model-00009-of-00009.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ccc3646cb64b3be0b95d9a599e5857613155a9a15168370edb301897f7b0a0a
+size 3696872248

model.safetensors.index.json

@@ -0,0 +1,179 @@
+{
+  "metadata": {
+    "total_size": 42124290048
+  },
+  "weight_map": {
+    "embed_out.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.embed_in.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.final_layer_norm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.final_layer_norm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.0.attention.dense.bias": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.attention.dense.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.attention.query_key_value.bias": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.attention.query_key_value.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.input_layernorm.bias": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.input_layernorm.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.0.mlp.dense_4h_to_h.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.bias": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.mlp.dense_h_to_4h.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.post_attention_layernorm.bias": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.0.post_attention_layernorm.weight": "model-00001-of-00009.safetensors",
+    "gpt_neox.layers.1.attention.dense.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.attention.dense.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.attention.query_key_value.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.attention.query_key_value.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.input_layernorm.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.input_layernorm.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.1.mlp.dense_4h_to_h.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.mlp.dense_h_to_4h.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.post_attention_layernorm.bias": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.1.post_attention_layernorm.weight": "model-00002-of-00009.safetensors",
+    "gpt_neox.layers.10.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.10.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.11.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.12.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.13.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.2.attention.dense.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.attention.dense.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.attention.query_key_value.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.attention.query_key_value.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.input_layernorm.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.input_layernorm.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.2.mlp.dense_4h_to_h.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.mlp.dense_h_to_4h.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.post_attention_layernorm.bias": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.2.post_attention_layernorm.weight": "model-00003-of-00009.safetensors",
+    "gpt_neox.layers.3.attention.dense.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.attention.dense.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.attention.query_key_value.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.attention.query_key_value.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.input_layernorm.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.input_layernorm.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.3.mlp.dense_4h_to_h.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.mlp.dense_h_to_4h.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.post_attention_layernorm.bias": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.3.post_attention_layernorm.weight": "model-00004-of-00009.safetensors",
+    "gpt_neox.layers.4.attention.dense.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.attention.dense.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.attention.query_key_value.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.attention.query_key_value.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.input_layernorm.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.input_layernorm.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.4.mlp.dense_4h_to_h.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.mlp.dense_h_to_4h.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.post_attention_layernorm.bias": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.4.post_attention_layernorm.weight": "model-00005-of-00009.safetensors",
+    "gpt_neox.layers.5.attention.dense.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.attention.dense.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.attention.query_key_value.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.attention.query_key_value.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.input_layernorm.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.input_layernorm.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.5.mlp.dense_4h_to_h.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.mlp.dense_h_to_4h.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.post_attention_layernorm.bias": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.5.post_attention_layernorm.weight": "model-00006-of-00009.safetensors",
+    "gpt_neox.layers.6.attention.dense.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.attention.dense.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.attention.query_key_value.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.attention.query_key_value.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.input_layernorm.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.input_layernorm.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.6.mlp.dense_4h_to_h.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.mlp.dense_h_to_4h.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.post_attention_layernorm.bias": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.6.post_attention_layernorm.weight": "model-00007-of-00009.safetensors",
+    "gpt_neox.layers.7.attention.dense.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.attention.dense.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.attention.query_key_value.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.attention.query_key_value.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.input_layernorm.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.input_layernorm.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.7.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.mlp.dense_h_to_4h.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.post_attention_layernorm.bias": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.7.post_attention_layernorm.weight": "model-00008-of-00009.safetensors",
+    "gpt_neox.layers.8.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.8.post_attention_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.attention.dense.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.attention.dense.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.attention.query_key_value.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.attention.query_key_value.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.input_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.input_layernorm.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.mlp.dense_4h_to_h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.mlp.dense_h_to_4h.weight": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.post_attention_layernorm.bias": "model-00009-of-00009.safetensors",
+    "gpt_neox.layers.9.post_attention_layernorm.weight": "model-00009-of-00009.safetensors"
+  }
+}