| { | |
| "format_version": 1, | |
| "checkpoint_type": "pretrain", | |
| "model_config": { | |
| "vocab_size": 1980, | |
| "max_seq_len": 512, | |
| "n_outcomes": 11, | |
| "d_model": 640, | |
| "n_layers": 10, | |
| "n_heads": 8, | |
| "d_ff": 2560, | |
| "dropout": 0.0, | |
| "rope_base": 10000.0 | |
| }, | |
| "training_config": { | |
| "lr": 0.0003, | |
| "weight_decay": 0.01, | |
| "max_grad_norm": 1.0, | |
| "warmup_steps": 10000, | |
| "total_steps": 200000, | |
| "batch_size": 256, | |
| "max_ply": 512, | |
| "discard_ply_limit": false, | |
| "num_workers": 4, | |
| "use_amp": true, | |
| "accumulation_steps": 1, | |
| "log_interval": 50, | |
| "eval_interval": 1000, | |
| "checkpoint_interval": 5000, | |
| "pause_after_steps": null, | |
| "no_outcome_token": false, | |
| "prepend_outcome": false, | |
| "mate_boost": 0.0, | |
| "base_seed": 42, | |
| "val_seed": 9223372036854775807, | |
| "val_games": 2048, | |
| "checkpoint_dir": "checkpoints", | |
| "log_dir": "/workspace/logs", | |
| "use_wandb": false, | |
| "wandb_project": "pawn", | |
| "device": "cuda" | |
| } | |
| } |