config.json

{
  "d": 512,
  "vocab_size": 49152,
  "num_heads": 8,
  "head_dim": 64,
  "mlp_hidden_dim": 1408,
  "diff_blocks_num": 3,
  "layers_per_block": 6,
  "rope_theta": 50000.0,
  "max_position_embeddings": 4096,
  "mask_token_id": 49151,
  "max_insert": 4,
  "rms_eps": 1e-06,
  "attn_backend": "sdpa"
}