{ "vocab_size": 16384, "d_model": 1024, "n_layers": 24, "max_seq_len": 4096, "attn_every": 4, "n_heads": 8, "n_kv_heads": 2, "head_dim": 128, "rope_theta": 50000.0, "dn_k_heads": 4, "dn_v_heads": 8, "dn_head_dim": 128, "dn_conv_size": 4, "ffn_hidden": 2816, "norm_eps": 1e-06, "tie_embeddings": true, "init_std": 0.02 }