| { | |
| "model": { | |
| "encoder_dim": 384, | |
| "encoder_type": "strengthened", | |
| "encoder_conv_blocks": 2, | |
| "normalize_stm": true, | |
| "hidden_dim": 512, | |
| "num_layers": 96, | |
| "num_heads": 8, | |
| "ffn_ratio": 4, | |
| "dropout": 0.05, | |
| "policy_head_dim": 384, | |
| "value_hidden": 384, | |
| "use_pos_embed": false, | |
| "n_ctx_tokens": 4, | |
| "value_head_type": "cls", | |
| "n_value_classes": 128, | |
| "use_swiglu": true, | |
| "use_rel_bias": true, | |
| "gradient_checkpointing": false | |
| }, | |
| "training": { | |
| "go": true, | |
| "cloud": false, | |
| "a100": true, | |
| "a100_309m": true, | |
| "force": true, | |
| "smoke": false, | |
| "steps": 50000, | |
| "batch_size": 192, | |
| "accum_steps": 1, | |
| "muon_lr": 0.02, | |
| "adam_lr": 0.0003, | |
| "min_lr_frac": 0.05, | |
| "warmup": 500, | |
| "value_weight": 0.5, | |
| "weight_decay": 0.01, | |
| "grad_clip": 1.0, | |
| "label_smoothing": 0.05, | |
| "log_interval": 25, | |
| "save_interval": 1000, | |
| "shard_dir": "/root/transform/outputs/exp139_massive_train/shards", | |
| "shuffle_buffer": 4096, | |
| "min_depth": 12, | |
| "resume": true, | |
| "output_dir": "outputs/exp182_pretrain_a40_309m", | |
| "use_bf16": true | |
| } | |
| } |