{
  "arc": {
    "num_heads": 8,
    "attention_dim": 768,
    "vocab_size": 16384,
    "num_blocks": 8,
    "ff_dim": 3072
  },
  "attn": {
    "attn_chunks": 4,
    "gqa_repeats": 2,
    "theeta": 10000
  },
  "moe": {
    "num_experts": 1,
    "top_k": 1,
    "capacity": 1.25
  },
  "dropout_rate": 0.03
}