config.json

{
  "_comment": "Geometry of model.cflow, read from its own .cflow v5 header. cflow is a standalone Rust runtime, not a transformers architecture: `model_type` and `architectures` are deliberately absent so nothing tries to load this with transformers. See README.md for how to run it.",

  "library_name": "cflow",
  "model_name": "arch2_4_8k_16l",
  "cflow_format_version": 5,

  "num_hidden_layers": 16,
  "hidden_size": 8192,
  "vocab_size": 50257,
  "tokenizer": "gpt2",

  "num_attention_heads": 128,
  "num_key_value_heads": 128,
  "head_dim": 64,
  "attention": "uniform",
  "sliding_window": null,
  "rope_theta": 10000.0,
  "rms_norm_eps": 1e-06,

  "intermediate_size": 32768,
  "hidden_act": "gelu_pytorch_tanh",
  "num_experts": 8,
  "num_experts_per_tok": 2,
  "moe_intermediate_size": 4096,

  "combine_style": "DelayedSum",
  "dense_delay": 1,
  "expert_delay": 2,

  "quantization": {
    "method": "cflow-q4_0",
    "bits": 4,
    "group_size": 32,
    "tile_rows": 128,
    "tile_cols": 256
  },

  "_notes": {
    "params": "~31B total, ~1.9B per layer",
    "rms_norm_eps": "not recorded in the v5 header; the runtime substitutes 1e-6, which is the value this model was trained with",
    "rope_theta": "not recorded in the v5 header; the runtime defaults to 10000, which is this model's base",
    "delays": "dense_delay=1 and expert_delay=2 are the pipeline-native schedule; they change the dependency graph, not just the order of execution"
  }
}