{
  "arc": {
    "num_heads": 12,
    "attention_dim": 768,
    "vocab_size": 43012,
    "num_blocks": 12,
    "ff_dim": 2304
  },
  "attn": {
    "attn_chunks": 2,
    "gqa_repeats": 3,
    "theeta": 10000
  },
  "moe": {
    "num_experts": 1,
    "top_k": 1,
    "capacity": 1.0
  },
  "dropout_rate": 0.05
}