{
  "vocab_size": 50264,
  "context_length": 512,
  "d_model": 512,
  "n_heads": 8,
  "n_layers": 6,
  "d_ff": 2048,
  "dropout": 0.1,
  "bias": true,
  "use_rope": true,
  "rope_base": 10000.0,
  "use_swiglu": true,
  "use_rmsnorm": true,
  "use_flash_attention": false,
  "n_kv_heads": null,
  "use_qk_norm": false,
  "intra_doc_mask": true,
  "eos_token_id": 50258,
  "use_moe": false,
  "n_experts": 4,
  "n_experts_active": 2,
  "moe_aux_loss_coef": 0.01,
  "batch_size": 16,
  "learning_rate": 0.0003,
  "weight_decay": 0.1,
  "beta1": 0.9,
  "beta2": 0.95,
  "grad_clip": 1.0,
  "grad_accum_steps": 4,
  "warmup_steps": 2000,
  "max_steps": 30000,
  "eval_interval": 1500,
  "save_interval": 3000,
  "data_dir": "data/processed",
  "checkpoint_dir": "checkpoints/ghost_small_gen",
  "log_dir": "logs/ghost_small_gen",
  "device": "mps",
  "dtype": "float32",
  "seed": 42,
  "use_wandb": false,
  "wandb_project": "ghostlm",
  "use_compile": false,
  "gradient_checkpointing": false
}