config.json

{
  "format_version": 1,
  "checkpoint_type": "pretrain",
  "model_config": {
    "vocab_size": 1980,
    "max_seq_len": 512,
    "n_outcomes": 11,
    "d_model": 256,
    "n_layers": 8,
    "n_heads": 4,
    "d_ff": 1024,
    "dropout": 0.0,
    "rope_base": 10000.0
  },
  "training_config": {
    "lr": 0.0003,
    "weight_decay": 0.01,
    "max_grad_norm": 1.0,
    "warmup_steps": 10000,
    "total_steps": 200000,
    "batch_size": 256,
    "max_ply": 512,
    "discard_ply_limit": false,
    "num_workers": 4,
    "use_amp": true,
    "accumulation_steps": 1,
    "log_interval": 50,
    "eval_interval": 1000,
    "checkpoint_interval": 5000,
    "pause_after_steps": null,
    "no_outcome_token": false,
    "prepend_outcome": false,
    "mate_boost": 0.0,
    "base_seed": 42,
    "val_seed": 9223372036854775807,
    "val_games": 2048,
    "checkpoint_dir": "checkpoints",
    "log_dir": "/workspace/logs",
    "use_wandb": false,
    "wandb_project": "pawn",
    "device": "cuda"
  }
}