| { | |
| "model": { | |
| "encoder_dim": 256, | |
| "encoder_type": "fused", | |
| "encoder_conv_blocks": 2, | |
| "normalize_stm": false, | |
| "hidden_dim": 1024, | |
| "num_layers": 16, | |
| "num_heads": 16, | |
| "ffn_ratio": 4, | |
| "dropout": 0.1, | |
| "policy_head_dim": 512, | |
| "value_hidden": 512, | |
| "use_pos_embed": true, | |
| "n_ctx_tokens": 4, | |
| "value_head_type": "cls", | |
| "n_value_classes": 3, | |
| "use_swiglu": false, | |
| "use_rel_bias": false, | |
| "gradient_checkpointing": false, | |
| "full_dim_attention": false | |
| }, | |
| "training": { | |
| "go": true, | |
| "smoke": false, | |
| "checkpoint": "outputs/hf_checkpoint/best_model.pt", | |
| "soft_cache": "outputs/exp186_sf_multipv/soft_cache_merged.pt", | |
| "output_dir": "outputs/exp188_200m_compact_soft", | |
| "steps": 5000, | |
| "batch_size": 384, | |
| "accum_steps": 2, | |
| "soft_frac": 0.85, | |
| "soft_alpha": 0.25, | |
| "lr": 1e-05, | |
| "resume": "outputs/exp188_200m_compact_soft/aborted.pt", | |
| "resume_opt": false, | |
| "min_lr_frac": 0.1, | |
| "warmup": 200, | |
| "value_weight": 0.1, | |
| "weight_decay": 0.01, | |
| "grad_clip": 1.0, | |
| "label_smoothing": 0.0, | |
| "log_interval": 25, | |
| "save_interval": 500, | |
| "eval_interval": 500, | |
| "min_depth": 12, | |
| "shuffle_buffer": 2048 | |
| }, | |
| "n_params": 204006404, | |
| "n_soft": 1987766, | |
| "vocab": "compact" | |
| } |