config.json

{
 "hidden_size": 4096,
 "intermediate_size": 12288,
 "num_layers": 36,
 "n_heads": 32,
 "n_kv_heads": 8,
 "head_dim": 128,
 "vocab_size": 151936,
 "rope_theta": 1000000.0,
 "rms_eps": 1e-06,
 "tie_embeddings": false,
 "qk_norm": true,
 "K": 3
}