config.json

{
  "model_type": "campgpt",
  "architectures": [
    "CampGPT"
  ],
  "vocab_size": 50304,
  "n_embd": 768,
  "n_head": 12,
  "n_kv_head": 4,
  "n_layer": 12,
  "block_size": 1024,
  "norm_eps": 1e-06,
  "multiple_of": 64,
  "use_moe": true,
  "n_experts": 3,
  "n_experts_per_tok": 1,
  "n_shared_experts": 1,
  "total_params": 322680576,
  "training_stages": [
    "pretrain_10B",
    "sft",
    "dpo"
  ],
  "val_loss": 0.0698780653377374
}