config.json

{
  "d_model": 1024,
  "n_heads": 8,
  "n_kv_heads": 4,
  "head_dim": 128,
  "n_layers": 18,
  "vocab_size": 262144,
  "ffn_hidden": 4096,
  "chunk_size": 8,
  "max_cache_chunks": 64,
  "n_modalities": 4,
  "n_tasks": 6,
  "n_languages": 4022,
  "n_mtp_heads": 4
}