Add config.json: model geometry, and the Hub query file that makes downloads countable
Browse files- config.json +45 -0
config.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_comment": "Geometry of model.cflow, read from its own .cflow v5 header. cflow is a standalone Rust runtime, not a transformers architecture: `model_type` and `architectures` are deliberately absent so nothing tries to load this with transformers. See README.md for how to run it.",
|
| 3 |
+
|
| 4 |
+
"library_name": "cflow",
|
| 5 |
+
"model_name": "arch2_4_8k_16l",
|
| 6 |
+
"cflow_format_version": 5,
|
| 7 |
+
|
| 8 |
+
"num_hidden_layers": 16,
|
| 9 |
+
"hidden_size": 8192,
|
| 10 |
+
"vocab_size": 50257,
|
| 11 |
+
"tokenizer": "gpt2",
|
| 12 |
+
|
| 13 |
+
"num_attention_heads": 128,
|
| 14 |
+
"num_key_value_heads": 128,
|
| 15 |
+
"head_dim": 64,
|
| 16 |
+
"attention": "uniform",
|
| 17 |
+
"sliding_window": null,
|
| 18 |
+
"rope_theta": 10000.0,
|
| 19 |
+
"rms_norm_eps": 1e-06,
|
| 20 |
+
|
| 21 |
+
"intermediate_size": 32768,
|
| 22 |
+
"hidden_act": "gelu_pytorch_tanh",
|
| 23 |
+
"num_experts": 8,
|
| 24 |
+
"num_experts_per_tok": 2,
|
| 25 |
+
"moe_intermediate_size": 4096,
|
| 26 |
+
|
| 27 |
+
"combine_style": "DelayedSum",
|
| 28 |
+
"dense_delay": 1,
|
| 29 |
+
"expert_delay": 2,
|
| 30 |
+
|
| 31 |
+
"quantization": {
|
| 32 |
+
"method": "cflow-q4_0",
|
| 33 |
+
"bits": 4,
|
| 34 |
+
"group_size": 32,
|
| 35 |
+
"tile_rows": 128,
|
| 36 |
+
"tile_cols": 256
|
| 37 |
+
},
|
| 38 |
+
|
| 39 |
+
"_notes": {
|
| 40 |
+
"params": "~31B total, ~1.9B per layer",
|
| 41 |
+
"rms_norm_eps": "not recorded in the v5 header; the runtime substitutes 1e-6, which is the value this model was trained with",
|
| 42 |
+
"rope_theta": "not recorded in the v5 header; the runtime defaults to 10000, which is this model's base",
|
| 43 |
+
"delays": "dense_delay=1 and expert_delay=2 are the pipeline-native schedule; they change the dependency graph, not just the order of execution"
|
| 44 |
+
}
|
| 45 |
+
}
|