tompoper commited on
Commit
aef80b6
·
verified ·
1 Parent(s): 4eb7765

Add config.json: model geometry, and the Hub query file that makes downloads countable

Browse files
Files changed (1) hide show
  1. config.json +45 -0
config.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_comment": "Geometry of model.cflow, read from its own .cflow v5 header. cflow is a standalone Rust runtime, not a transformers architecture: `model_type` and `architectures` are deliberately absent so nothing tries to load this with transformers. See README.md for how to run it.",
3
+
4
+ "library_name": "cflow",
5
+ "model_name": "arch2_4_8k_16l",
6
+ "cflow_format_version": 5,
7
+
8
+ "num_hidden_layers": 16,
9
+ "hidden_size": 8192,
10
+ "vocab_size": 50257,
11
+ "tokenizer": "gpt2",
12
+
13
+ "num_attention_heads": 128,
14
+ "num_key_value_heads": 128,
15
+ "head_dim": 64,
16
+ "attention": "uniform",
17
+ "sliding_window": null,
18
+ "rope_theta": 10000.0,
19
+ "rms_norm_eps": 1e-06,
20
+
21
+ "intermediate_size": 32768,
22
+ "hidden_act": "gelu_pytorch_tanh",
23
+ "num_experts": 8,
24
+ "num_experts_per_tok": 2,
25
+ "moe_intermediate_size": 4096,
26
+
27
+ "combine_style": "DelayedSum",
28
+ "dense_delay": 1,
29
+ "expert_delay": 2,
30
+
31
+ "quantization": {
32
+ "method": "cflow-q4_0",
33
+ "bits": 4,
34
+ "group_size": 32,
35
+ "tile_rows": 128,
36
+ "tile_cols": 256
37
+ },
38
+
39
+ "_notes": {
40
+ "params": "~31B total, ~1.9B per layer",
41
+ "rms_norm_eps": "not recorded in the v5 header; the runtime substitutes 1e-6, which is the value this model was trained with",
42
+ "rope_theta": "not recorded in the v5 header; the runtime defaults to 10000, which is this model's base",
43
+ "delays": "dense_delay=1 and expert_delay=2 are the pipeline-native schedule; they change the dependency graph, not just the order of execution"
44
+ }
45
+ }