{
  "model": {
    "encoder_dim": 384,
    "encoder_type": "strengthened",
    "encoder_conv_blocks": 2,
    "normalize_stm": true,
    "hidden_dim": 512,
    "num_layers": 96,
    "num_heads": 8,
    "ffn_ratio": 4,
    "dropout": 0.05,
    "policy_head_dim": 384,
    "value_hidden": 384,
    "use_pos_embed": false,
    "n_ctx_tokens": 4,
    "value_head_type": "cls",
    "n_value_classes": 128,
    "use_swiglu": true,
    "use_rel_bias": true,
    "gradient_checkpointing": false
  },
  "training": {
    "go": true,
    "cloud": false,
    "a100": true,
    "a100_309m": true,
    "force": true,
    "smoke": false,
    "steps": 50000,
    "batch_size": 192,
    "accum_steps": 1,
    "muon_lr": 0.02,
    "adam_lr": 0.0003,
    "min_lr_frac": 0.05,
    "warmup": 500,
    "value_weight": 0.5,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "label_smoothing": 0.05,
    "log_interval": 25,
    "save_interval": 1000,
    "shard_dir": "/root/transform/outputs/exp139_massive_train/shards",
    "shuffle_buffer": 4096,
    "min_depth": 12,
    "resume": true,
    "output_dir": "outputs/exp182_pretrain_a40_309m",
    "use_bf16": true
  }
}