model_cfg.json

{
  "core_model": {
    "num_layers": 8,
    "ffn": {
      "ffn_type": "swiglu",
      "ffn_dim": 1320,
      "normalization": "rms_norm",
      "bias": false
    },
    "attn": {
      "attn_type": "generic",
      "num_heads": 16,
      "normalization": "rms_norm",
      "group_size": 4,
      "bias": false,
      "is_causal": true
    }
  },
  "lm_head": {
    "normalization": "rms_norm",
    "bias": false
  },
  "hidden_dim": 512,
  "context_window": 512,
  "vocab_size": 50257,
  "embedding_weight_tying": true,
  "positional_encoding_type": "rope",
  "checkpoint_name": null
}