config.json

{
  "vocab_size": 32832,
  "n_layer": 24,
  "d_model": 1024,
  "arch": "hybrid",
  "attn_every": 4,
  "n_head": 16,
  "n_kv_head": 4,
  "head_dim": 128,
  "gdn_heads": 16,
  "gdn_head_dim": 128,
  "ffn_dim": 3584,
  "attnres_blocks": 8,
  "rope_theta": 10000.0,
  "max_seq_len": 4096,
  "softcap": 15.0,
  "grad_ckpt": false
}