Commit ·
1b7abd9
1
Parent(s): bf11a55
Upload CustomTransformerForCausalLM
Browse files- config.json +3 -11
- model.safetensors.index.json +2 -2
config.json
CHANGED
|
@@ -23,16 +23,8 @@
|
|
| 23 |
"num_attention_heads": 40,
|
| 24 |
"num_hidden_layers": 40,
|
| 25 |
"num_key_value_heads": 10,
|
| 26 |
-
"pr_chunk_size":
|
| 27 |
-
|
| 28 |
-
2048
|
| 29 |
-
]
|
| 30 |
-
],
|
| 31 |
-
"pr_degree": [
|
| 32 |
-
[
|
| 33 |
-
2
|
| 34 |
-
]
|
| 35 |
-
],
|
| 36 |
"rms_norm_eps": 1e-05,
|
| 37 |
"rope_scaling": {
|
| 38 |
"factor": 31.25,
|
|
@@ -45,5 +37,5 @@
|
|
| 45 |
"use_cache": true,
|
| 46 |
"use_gated_attention": true,
|
| 47 |
"use_gated_delta_net": true,
|
| 48 |
-
"vocab_size":
|
| 49 |
}
|
|
|
|
| 23 |
"num_attention_heads": 40,
|
| 24 |
"num_hidden_layers": 40,
|
| 25 |
"num_key_value_heads": 10,
|
| 26 |
+
"pr_chunk_size": 2048,
|
| 27 |
+
"pr_degree": 2,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
"rms_norm_eps": 1e-05,
|
| 29 |
"rope_scaling": {
|
| 30 |
"factor": 31.25,
|
|
|
|
| 37 |
"use_cache": true,
|
| 38 |
"use_gated_attention": true,
|
| 39 |
"use_gated_delta_net": true,
|
| 40 |
+
"vocab_size": 200019
|
| 41 |
}
|
model.safetensors.index.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
{
|
| 2 |
"metadata": {
|
| 3 |
-
"total_parameters":
|
| 4 |
-
"total_size":
|
| 5 |
},
|
| 6 |
"weight_map": {
|
| 7 |
"model.embed_tokens.weight": "model-00001-of-00003.safetensors",
|
|
|
|
| 1 |
{
|
| 2 |
"metadata": {
|
| 3 |
+
"total_parameters": 3088044840,
|
| 4 |
+
"total_size": 12352179360
|
| 5 |
},
|
| 6 |
"weight_map": {
|
| 7 |
"model.embed_tokens.weight": "model-00001-of-00003.safetensors",
|