{
  "model": {
    "encoder_dim": 256,
    "encoder_type": "fused",
    "encoder_conv_blocks": 2,
    "normalize_stm": false,
    "hidden_dim": 1024,
    "num_layers": 16,
    "num_heads": 16,
    "ffn_ratio": 4,
    "dropout": 0.1,
    "policy_head_dim": 512,
    "value_hidden": 512,
    "use_pos_embed": true,
    "n_ctx_tokens": 4,
    "value_head_type": "cls",
    "n_value_classes": 3,
    "use_swiglu": false,
    "use_rel_bias": false,
    "gradient_checkpointing": false,
    "full_dim_attention": false
  },
  "training": {
    "go": true,
    "smoke": false,
    "checkpoint": "outputs/hf_checkpoint/best_model.pt",
    "soft_cache": "outputs/exp186_sf_multipv/soft_cache_merged.pt",
    "output_dir": "outputs/exp188_200m_compact_soft",
    "steps": 5000,
    "batch_size": 384,
    "accum_steps": 2,
    "soft_frac": 0.85,
    "soft_alpha": 0.25,
    "lr": 1e-05,
    "resume": "outputs/exp188_200m_compact_soft/aborted.pt",
    "resume_opt": false,
    "min_lr_frac": 0.1,
    "warmup": 200,
    "value_weight": 0.1,
    "weight_decay": 0.01,
    "grad_clip": 1.0,
    "label_smoothing": 0.0,
    "log_interval": 25,
    "save_interval": 500,
    "eval_interval": 500,
    "min_depth": 12,
    "shuffle_buffer": 2048
  },
  "n_params": 204006404,
  "n_soft": 1987766,
  "vocab": "compact"
}