checkpoints/step_00019638/training_meta.json

{
  "step": 19638,
  "epoch": 0,
  "total_tokens": 1286995968,
  "val_loss": 2.6220367693901063,
  "config": {
    "vocab_size": 50304,
    "d_model": 4096,
    "n_layers": 56,
    "n_heads": 32,
    "n_kv_heads": 8,
    "max_len": 2048,
    "rope_theta": 5000000.0,
    "ffn_hidden": 11008,
    "use_moe": false,
    "moe_experts": 8,
    "moe_top_k": 2,
    "moe_shared": 1,
    "moe_hidden": 1408,
    "moe_every": 4,
    "moe_bias_speed": 0.001,
    "use_qk_norm": true,
    "use_structural_bias": true,
    "n_struct_rel": 4,
    "use_fim": true,
    "fim_rate": 0.5,
    "fim_pre_id": 50269,
    "fim_suf_id": 50270,
    "fim_mid_id": 50271,
    "grad_checkpointing": true,
    "lr": 0.0001,
    "min_lr": 1e-05,
    "weight_decay": 0.1,
    "beta1": 0.9,
    "beta2": 0.95,
    "grad_clip": 1.0,
    "warmup_tokens": 500000,
    "target_total_tokens": 10000000000,
    "micro_batch": 4,
    "grad_accum": 8,
    "log_every": 10,
    "ckpt_every_steps": 200,
    "eval_every_steps": 200,
    "seed": 42,
    "max_session_hours": 11.5
  },
  "timestamp": 1785469962.5254455
}