| { |
| "_comment": "Geometry of model.cflow, read from its own .cflow v5 header. cflow is a standalone Rust runtime, not a transformers architecture: `model_type` and `architectures` are deliberately absent so nothing tries to load this with transformers. See README.md for how to run it.", |
|
|
| "library_name": "cflow", |
| "model_name": "arch2_4_8k_16l", |
| "cflow_format_version": 5, |
|
|
| "num_hidden_layers": 16, |
| "hidden_size": 8192, |
| "vocab_size": 50257, |
| "tokenizer": "gpt2", |
|
|
| "num_attention_heads": 128, |
| "num_key_value_heads": 128, |
| "head_dim": 64, |
| "attention": "uniform", |
| "sliding_window": null, |
| "rope_theta": 10000.0, |
| "rms_norm_eps": 1e-06, |
|
|
| "intermediate_size": 32768, |
| "hidden_act": "gelu_pytorch_tanh", |
| "num_experts": 8, |
| "num_experts_per_tok": 2, |
| "moe_intermediate_size": 4096, |
|
|
| "combine_style": "DelayedSum", |
| "dense_delay": 1, |
| "expert_delay": 2, |
|
|
| "quantization": { |
| "method": "cflow-q4_0", |
| "bits": 4, |
| "group_size": 32, |
| "tile_rows": 128, |
| "tile_cols": 256 |
| }, |
|
|
| "_notes": { |
| "params": "~31B total, ~1.9B per layer", |
| "rms_norm_eps": "not recorded in the v5 header; the runtime substitutes 1e-6, which is the value this model was trained with", |
| "rope_theta": "not recorded in the v5 header; the runtime defaults to 10000, which is this model's base", |
| "delays": "dense_delay=1 and expert_delay=2 are the pipeline-native schedule; they change the dependency graph, not just the order of execution" |
| } |
| } |
|
|
